From: <fko...@us...> - 2010-09-08 20:09:19
|
Revision: 3521 http://bigdata.svn.sourceforge.net/bigdata/?rev=3521&view=rev Author: fkoliver Date: 2010-09-08 20:09:11 +0000 (Wed, 08 Sep 2010) Log Message: ----------- Remove dependency on cweb-junit-*.jar by copying the sources into this project. This is a temporary step: the extensions should be eventually be removed as we move to junit 4.x. Modified Paths: -------------- branches/maven_scaleout/bigdata-core/pom.xml branches/maven_scaleout/bigdata-core/src/test/deploy/testing/conf/standalone/ServerStarter.config branches/maven_scaleout/bigdata-core/thirdparty/maven.xml Added Paths: ----------- branches/maven_scaleout/bigdata-core/src/test/java/junit/ branches/maven_scaleout/bigdata-core/src/test/java/junit/extensions/ branches/maven_scaleout/bigdata-core/src/test/java/junit/extensions/proxy/ branches/maven_scaleout/bigdata-core/src/test/java/junit/extensions/proxy/IProxyTest.java branches/maven_scaleout/bigdata-core/src/test/java/junit/extensions/proxy/ProxyTestSuite.java branches/maven_scaleout/bigdata-core/src/test/java/junit/extensions/proxy/package.html branches/maven_scaleout/bigdata-core/src/test/java/junit/framework/ branches/maven_scaleout/bigdata-core/src/test/java/junit/framework/TestCase2.java branches/maven_scaleout/bigdata-core/src/test/java/junit/framework/package.html Removed Paths: ------------- branches/maven_scaleout/bigdata-core/thirdparty/lib/cweb-junit-ext-1.1-b3-dev.jar Property Changed: ---------------- branches/maven_scaleout/bigdata-core/ Property changes on: branches/maven_scaleout/bigdata-core ___________________________________________________________________ Added: svn:ignore + target Modified: branches/maven_scaleout/bigdata-core/pom.xml =================================================================== --- branches/maven_scaleout/bigdata-core/pom.xml 2010-09-08 14:52:29 UTC (rev 3520) +++ branches/maven_scaleout/bigdata-core/pom.xml 2010-09-08 20:09:11 UTC (rev 3521) @@ -216,12 +216,6 @@ </dependency> <dependency> <groupId>${thirdParty.groupId}</groupId> - <artifactId>cweb-junit-ext</artifactId> - <version>1.1.0-b3-dev</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>${thirdParty.groupId}</groupId> <artifactId>cweb-commons</artifactId> <version>1.1.0-b2-dev</version> </dependency> Modified: branches/maven_scaleout/bigdata-core/src/test/deploy/testing/conf/standalone/ServerStarter.config =================================================================== --- branches/maven_scaleout/bigdata-core/src/test/deploy/testing/conf/standalone/ServerStarter.config 2010-09-08 14:52:29 UTC (rev 3520) +++ branches/maven_scaleout/bigdata-core/src/test/deploy/testing/conf/standalone/ServerStarter.config 2010-09-08 20:09:11 UTC (rev 3521) @@ -59,7 +59,6 @@ libdir+"icu4j-3_6.jar"+File.pathSeparator+ // test suites only! libdir+"junit-3.8.1.jar"+File.pathSeparator+ - libdir+"cweb-junit-ext-1.1-b2-dev.jar"+File.pathSeparator+ // main bigdata JAR. //libdir+ "bigdata-core.jar" Added: branches/maven_scaleout/bigdata-core/src/test/java/junit/extensions/proxy/IProxyTest.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/test/java/junit/extensions/proxy/IProxyTest.java (rev 0) +++ branches/maven_scaleout/bigdata-core/src/test/java/junit/extensions/proxy/IProxyTest.java 2010-09-08 20:09:11 UTC (rev 3521) @@ -0,0 +1,77 @@ +/** + +The Notice below must appear in each file of the Source Code of any +copy you distribute of the Licensed Product. Contributors to any +Modifications may add their own copyright notices to identify their +own contributions. + +License: + +The contents of this file are subject to the CognitiveWeb Open Source +License Version 1.1 (the License). You may not copy or use this file, +in either source code or executable form, except in compliance with +the License. You may obtain a copy of the License from + + http://www.CognitiveWeb.org/legal/license/ + +Software distributed under the License is distributed on an AS IS +basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +the License for the specific language governing rights and limitations +under the License. + +Copyrights: + +Portions created by or assigned to CognitiveWeb are Copyright +(c) 2003-2003 CognitiveWeb. All Rights Reserved. Contact +information for CognitiveWeb is available at + + http://www.CognitiveWeb.org + +Portions Copyright (c) 2002-2003 Bryan Thompson. + +Acknowledgements: + +Special thanks to the developers of the Jabber Open Source License 1.0 +(JOSL), from which this License was derived. This License contains +terms that differ from JOSL. + +Special thanks to the CognitiveWeb Open Source Contributors for their +suggestions and support of the Cognitive Web. + +Modifications: + +*/ + +package junit.extensions.proxy; + +import junit.framework.*; + +/** + * A {@link Test} that holds a reference to a delegate. Normally the + * delegate will extend {@link TestCase} to provide, possibly + * persistent, implementation specific logic for establishing test + * fixtures. The delegate is normally an instance of your concrete + * implementation specific test harness with access to any required + * configuration data, e.g., a properties file. + */ + +public interface IProxyTest + extends Test +{ + + /** + * Sets the delegate. {@link ProxyTestSuite} uses this method to + * set the delegate on each test class instance that it creates + * that implements the {@link IProxyTest} interface. + */ + + public void setDelegate( Test delegate ); + + /** + * Returns the reference to the delegate or <code>null</code> if + * the delegate was not established. + */ + + public Test getDelegate(); + +} Property changes on: branches/maven_scaleout/bigdata-core/src/test/java/junit/extensions/proxy/IProxyTest.java ___________________________________________________________________ Added: svn:keywords + Id Date Revision Author HeadURL Added: branches/maven_scaleout/bigdata-core/src/test/java/junit/extensions/proxy/ProxyTestSuite.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/test/java/junit/extensions/proxy/ProxyTestSuite.java (rev 0) +++ branches/maven_scaleout/bigdata-core/src/test/java/junit/extensions/proxy/ProxyTestSuite.java 2010-09-08 20:09:11 UTC (rev 3521) @@ -0,0 +1,395 @@ +/** + +The Notice below must appear in each file of the Source Code of any +copy you distribute of the Licensed Product. Contributors to any +Modifications may add their own copyright notices to identify their +own contributions. + +License: + +The contents of this file are subject to the CognitiveWeb Open Source +License Version 1.1 (the License). You may not copy or use this file, +in either source code or executable form, except in compliance with +the License. You may obtain a copy of the License from + + http://www.CognitiveWeb.org/legal/license/ + +Software distributed under the License is distributed on an AS IS +basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +the License for the specific language governing rights and limitations +under the License. + +Copyrights: + +Portions created by or assigned to CognitiveWeb are Copyright +(c) 2003-2003 CognitiveWeb. All Rights Reserved. Contact +information for CognitiveWeb is available at + + http://www.CognitiveWeb.org + +Portions Copyright (c) 2002-2003 Bryan Thompson. + +Acknowledgements: + +Special thanks to the developers of the Jabber Open Source License 1.0 +(JOSL), from which this License was derived. This License contains +terms that differ from JOSL. + +Special thanks to the CognitiveWeb Open Source Contributors for their +suggestions and support of the Cognitive Web. + +Modifications: + +*/ + +package junit.extensions.proxy; + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; + +import org.apache.log4j.Logger; + +/** + * <p> + * A simple wrapper around {@link TestSuite} that permits the caller to specify + * the delegate {@link Test} for either directly or recursively contained + * {@link IProxyTest}s added to a {@link ProxyTestSuite}. There are three cases + * for junit: + * <ol> + * <li>An instantiated test. This is an instance of some class that extends + * {@link TestCase}. If the class implements {@link IProxyTest} then the + * <i>delegate </i> will be set on the test instance and will be available when + * that test runs.</li> + * <li>A test case. This is a class named to + * {@link TestSuite#addTestSuite(java.lang.Class)}. That method scans the test + * case class and generates an instantiated test for each conforming test method + * identified in the test case. The delegate is then set per above.</li> + * <li>A test suite. The delegate declared in a {@link ProxyTestSuite} + * constructor will be flowed down to each test added either directly or + * indirectly to that {@link ProxyTestSuite}. The various constructors all + * invoke this method to flow down the delegate. In addition, the + * {@link #addTest(Test)} and {@link #addTestSuite(Class)} methods also + * invoke this method to flow down the delegate to instantiated tests that + * implement {@link IProxyTest}.</li> + * </ol> + * </p> + * <p> + * The only case when flow down does not occur is when you have created a + * standard test suite, addeded it to a proxy test suite, and <em>then</em> + * you add additional tests to the standard test suite. There is no event + * notification model in junit and this action is not noticed by the {@link + * ProxyTestSuite}. + * </p> + */ + +public class ProxyTestSuite + extends TestSuite +{ + + /** + * The {@link Logger} is named for this class. + */ + + protected static final Logger log = Logger.getLogger + ( ProxyTestSuite.class + ); + + /** + * The delegate that will be used by each {@link IProxyTest}added to this + * {@link ProxyTestSuite}. + */ + + private final Test m_delegate; + + /** + * <p> + * Invoked automatically by the constructors. + * </p> + * + * @exception IllegalArgumentException + * if the <i>delegate </i> is <code>null</code>. + */ + + final private void checkDelegate( Test delegate ) + { + + if (delegate == null) { + + throw new IllegalArgumentException("The delegate is null."); + + } + +// if (m_delegate != null) { +// +// throw new IllegalStateException("The delegate is already set."); +// +// } + + log.debug + ( "Delegate is "+delegate.getClass().getName() + ); + + } + + /** + * Returns the <i>delegate </i> supplied to the constructor. + */ + + public Test getDelegate() { + + return m_delegate; + + } + + /** + * Creates an empty unnamed test suite. The <i>delegate </i> will be + * assigned to tests added to this test suite that implement + * {@link IProxyTest}. + * + * @param delegate + * The delegate (non-null). + */ + + public ProxyTestSuite( Test delegate ) + { + + super(); + + checkDelegate( delegate ); + + m_delegate = delegate; + + } + + /** + * Creates a named test suite and populates it with test instances + * identified by scanning the <i>testClass </i>. The <i>delegate </i> will + * be assigned to tests added to this test suite that implement + * {@link IProxyTest}, including those created from <i>testClass </i>. + * + * @param delegate + * The delegate (non-null). + * @param testClass + * A class containing one or more tests. + * @param name + * The name of the test suite (optional). + */ + + public ProxyTestSuite( Test delegate, Class testClass, String name ) + { + + /* + * Let the super class do all the heavy lifting. Note that it can not set + * the delegate for us since we can not set our private fields until after + * we have invoked the super class constructor. + */ + + super( testClass, name ); + + checkDelegate( delegate ); + + m_delegate = delegate; + + /* + * Apply delegate to all instantiated tests that were created by + * junit from testClass and which implement IProxyTest. + */ + flowDown(this); + + } + + /** + * Creates an unnamed test suite and populates it with test instances + * identified by scanning the <i>testClass </i>. The <i>delegate </i> will + * be assigned to tests added to this test suite that implement + * {@link IProxyTest}, including those created from <i>testClass </i>. + * + * @param delegate + * The delegate (non-null). + * @param testClass + * A class containing one or more tests. + */ + + public ProxyTestSuite( Test delegate, Class testClass ) + { + + /* + * Let the super class do all the heavy lifting. Note that it can not set + * the delegate for us since we can not set our private fields until after + * we have invoked the super class constructor. + */ + + super( testClass ); + + // Remember the delegate. + + checkDelegate( delegate ); + + m_delegate = delegate; + + /* + * Apply delegate to all instantiated tests that were created by + * junit from testClass and which implement IProxyTest. + */ + flowDown(this); + + } + + /** + * Creates an empty named test suite. The declared will be assigned to tests + * added to this test suite that implement {@link IProxyTest}. + * + * @param delegate + * The delegate (non-null). + * @param name + * The test suite name (optional). + */ + public ProxyTestSuite( Test delegate, String name ) + { + + super( name ); + + checkDelegate( delegate ); + + m_delegate = delegate; + + } + + /** + * We override the implementation of {@link + * TestSuite#addTestSuite( Class theClass )} to wrap the + * <i>testClass</i> in another instance of this {@link + * ProxyTestSuite} class using the same delegate that was provided + * to our constructor. This causes the delegation to be inherited + * by any recursively contained <i>testClass</i> which implements + * {@link IProxyTest}. + */ + + public void addTestSuite( Class testClass ) + { + + if( m_delegate == null ) { + + /* + * The delegate will be null if this method gets invoked from the + * super class constructor since that will happen before we have a + * chance to set the [m_delegate] field. The constructors handle + * this situation by explictly flowing down the delegate after the + * super class constructor has been invoked. + */ + + super.addTestSuite( testClass ); + + return; + + } + + // This is the default implementation from TestSuite: + // + // addTest(new TestSuite(testClass)); + // + // Our implementation just substitutes new ProxyTestSuite( + // delegate, testClass ) for new TestSuite( testClass ). + // + + ProxyTestSuite proxyTestSuite = new ProxyTestSuite + ( m_delegate, + testClass + ); + + addTest( proxyTestSuite ); + + } + + /** + * If the suite is not a {@link ProxyTestSuite}, then the tests in the + * suite are recursively enumerated and a proxy test suite is created with + * the same name and tests. This ensures that the common delegate flows down + * through all tests even when using the traditional + * <code>static Test suite() {...}</code> construction. + * + * @param test + * A test. + */ + public void addTest(Test test) { + + if( m_delegate == null ) { + + /* + * The delegate will be null if this method gets invoked from the + * super class constructor since that will happen before we have a + * chance to set the [m_delegate] field. The constructors handle + * this situation by explictly flowing down the delegate after the + * super class constructor has been invoked. + */ + + super.addTest( test ); + + return; + + } + + if ( ! (test instanceof ProxyTestSuite)) { + + /* + * Flow down the delegate to any container IProxyTest instances. + */ + + flowDown( test ); + + } + + super.addTest( test ); + + } + + protected void flowDown(Test t) { + + if (m_delegate == null) { + + throw new AssertionError("delegate is not set."); + + } + + if ( t instanceof TestSuite ) { + + flowDown( (TestSuite) t ); + + } else if (t instanceof IProxyTest) { + + log.debug("Setting delegate on " + t.getClass() + " to " + + m_delegate.getClass()); + + ((IProxyTest) t).setDelegate(m_delegate); + + } + + } + + /** + * <p> + * Sets the delegate on each instantiated {@link Test} that implements + * {@link IProxyTest}. + * </p> + */ + + protected void flowDown( TestSuite suite ) { + + if( m_delegate == null ) { + + throw new AssertionError("delegate is not set."); + + } + + for( java.util.Enumeration e= suite.tests(); e.hasMoreElements(); ) { + + Test t = (Test)e.nextElement(); + + flowDown( t ); + + } + + } + +} Property changes on: branches/maven_scaleout/bigdata-core/src/test/java/junit/extensions/proxy/ProxyTestSuite.java ___________________________________________________________________ Added: svn:keywords + Id Date Revision Author HeadURL Added: branches/maven_scaleout/bigdata-core/src/test/java/junit/extensions/proxy/package.html =================================================================== --- branches/maven_scaleout/bigdata-core/src/test/java/junit/extensions/proxy/package.html (rev 0) +++ branches/maven_scaleout/bigdata-core/src/test/java/junit/extensions/proxy/package.html 2010-09-08 20:09:11 UTC (rev 3521) @@ -0,0 +1,146 @@ +<HTML> +<HEAD> +<TITLE>Proxy test suites</TITLE> +</HEAD> +<BODY> +<H1>Integration testing with proxy test suites.</H1> +<P>This package provides support for suites of <EM>proxy</EM> tests that +share a common test <EM>delegate</EM>. Normally the proxy tests and the +delegate will implement a common interface. The implementation of the +interface on the proxy test base class should route the behaviors to the +delegate test. This mechanism can be used to run an entire test suite +against a specific configuration for integration testing. Since the same +delegate instance is used for all tests in the suite, you can do things +such as establish a shared database connection that is used for the +entire test suite, i.e., you can do <EM>integration testing</EM>.</P> +<H2>Defining a configuration using getProperties()</H2> +<P>The pattern that I have found to be most useful for integration +testing of different configurations is to define a <CODE>public +Properties getProperties()</CODE> method that is implemented by the <EM>delegate</EM> +and accessed by various integration test set up and tear down methods +implemented by the delegate. For example:</P> +<PRE> + public Properties getProperties() { + if( m_properties == null ) { + /* + * Read and cache System properties. + */ + m_properties = System.getProperties(); + } + /* + * Wrap up the cached properties so that they are not modifable by the + * caller (no side effects between calls). + */ + return new Properties( m_properties ); + } +</PRE> +<p>This approach reads properties from the environement, so you can +define property values using JVM command line arguments <CODE>-Dname=value</CODE>. +You can also extend the {@link junit.framework.TestCase2} class and use +its <CODE>getProperties()</CODE> method, in which case properties are +read from a hierarchy of sources, including a file with a <CODE>.properties</CODE> +extension and the same base name as the delegate test class.</p> +<H2>Sharing a connection</H2> +<P>In order to setup variables, such as a JDBC connection, that are +reused by each test in the suite, you need to declare those variables as +fields on the <EM>delegate</EM> implementation class and wrap the test +suite in a {@link junit.framework.TestDecorator} as follows.</P> +<pre> + public static Test suite() { + + /* + * Create the delegate instance that will be reused by all proxy tests in + * the test suite. The variable is declared as 'final' so that we can + * access it from the anonymous TestDecorator class, below. + */ + final DefaultTestCase delegate = new DefaultTestCase(); // !!!! THIS CLASS !!!! + + /* + * Use a proxy test suite and specify the delegate. + */ + + ProxyTestSuite suite = new ProxyTestSuite(delegate); + + /* + * Add some tests to the test suite. The setDelegate() method will be + * called on each test that implements IProxyTest and passed the value + * provided to the ProxyTestSuite constructor above. + */ + suite.addTestCase( MyTestCase1.class ); + + // ... more tests. + + /* + * Wrap up in decorator that connects to the database before the test + * suite is run and then disconnects from the database afterwards. + */ + return new TestDecorator(suite) { + + private boolean disconnectAfter = false; + + public void basicRun(TestResult result) { + if (!delegate.isConnected()) { + delegate.connect(); + disconnectAfter = true; + } + super.basicRun(result); + if (disconnectAfter) { + delegate.disconnect(); + } + } + + }; +</pre> +<P>While the life span of the <EM>delegate</EM> spans that of the test +suite, the <CODE>setUp</CODE> and <CODE>tearDown</CODE> methods of the +delegate are invoked before and after each test <EM>implemented by the +delegate</EM>. Since the delegate does not generally implement tests +directly, those set up and tear down methods can not be used to perform +integration testing.</P> +<H2>Running or debugging a single test</H2> +<P>By its nature, a test that implements {@link IProxyTest} requires a <em>delegate</em> +in order to run. This works fine when you are running an entire test +suite defined by the delegate, but you have to take an extra step in +order to be able to run or debug a single test. The following code gives +an example of how you can deal with this problem. It defines a property +<em>testClass</em> whose value is the name of the delegate class. If the +delegate was not set (as will be the case if you try to run a single +test directly), then this property must be specified as a JVM argument +using <code>-DtestClass=....</code> and an instance of your delegate +will be created and used to run the proxy test.</P> +<P>If you test relies on a shared variable such as a JDBC connection, +then you still have to provide for that. One way to handle this is to +delegate the <code>setUp</code> and <code>tearDown</code> methods on the +proxy test. The delegate implementation of those methods would then +establish and close the Connection iff the Connection was not already +established.</P> + +<pre> + public Test getDelegate() { + + if (m_delegate == null) { + + /* + * This property gives the class name of the concrete instance of + * testClass that we need to instantiate so that we can run or debug + * a single test at a time. + */ + String testClass = getProperties().getProperty("testClass"); + if (testClass == null) { + + throw new IllegalStateException( + "testClass: property not defined, could not configure delegate."); + + } + try { + Class cl = Class.forName(testClass); + m_delegate = (Test) cl.newInstance(); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + + } + } +</pre> +</BODY> +</HTML> Property changes on: branches/maven_scaleout/bigdata-core/src/test/java/junit/extensions/proxy/package.html ___________________________________________________________________ Added: svn:keywords + Id Date Revision Author HeadURL Added: branches/maven_scaleout/bigdata-core/src/test/java/junit/framework/TestCase2.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/test/java/junit/framework/TestCase2.java (rev 0) +++ branches/maven_scaleout/bigdata-core/src/test/java/junit/framework/TestCase2.java 2010-09-08 20:09:11 UTC (rev 3521) @@ -0,0 +1,2230 @@ +/** + +The Notice below must appear in each file of the Source Code of any +copy you distribute of the Licensed Product. Contributors to any +Modifications may add their own copyright notices to identify their +own contributions. + +License: + +The contents of this file are subject to the CognitiveWeb Open Source +License Version 1.1 (the License). You may not copy or use this file, +in either source code or executable form, except in compliance with +the License. You may obtain a copy of the License from + + http://www.CognitiveWeb.org/legal/license/ + +Software distributed under the License is distributed on an AS IS +basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +the License for the specific language governing rights and limitations +under the License. + +Copyrights: + +Portions created by or assigned to CognitiveWeb are Copyright +(c) 2003-2003 CognitiveWeb. All Rights Reserved. Contact +information for CognitiveWeb is available at + + http://www.CognitiveWeb.org + +Portions Copyright (c) 2002-2003 Bryan Thompson. + +Acknowledgements: + +Special thanks to the developers of the Jabber Open Source License 1.0 +(JOSL), from which this License was derived. This License contains +terms that differ from JOSL. + +Special thanks to the CognitiveWeb Open Source Contributors for their +suggestions and support of the Cognitive Web. + +Modifications: + +*/ + +package junit.framework; + +// Test resource loader. +import java.io.Externalizable; +import java.io.Reader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.IOException; +import java.io.Serializable; + +// Properties hierarchy loader. +import java.io.File; +import java.io.FileInputStream; +import java.util.Properties; +import java.util.Random; +import java.util.TreeMap; +import java.util.Map; +import java.util.Iterator; + +// Logger. +import org.CognitiveWeb.util.PropertyUtil; +import org.apache.log4j.Logger; +import org.apache.log4j.Level; + +/** + * Extension of {@link TestCase} that supports logging, loading test + * resources, and hierarchical properties.<p> + * + * Note: When using Maven (or Ant), you need to be aware that that + * your <code>project.xml</code> has an impact on which files are + * recognized as tests to be submitted to a test runner. For example, + * it is common practice that all files beginning or ending with + * "Test" will be submitted to JUnit, though I suggest that the + * practice of naming things "FooTestCase" will serve you better. + * Also note that the maven test plugin explictly skips files whose + * name matches the pattern "*AbstractTestSuite".<p> + */ + +abstract public class TestCase2 + extends TestCase +{ + + public TestCase2() { + super(); + } + + public TestCase2(String name) { + super(name); + } + + /** + * The default {@link Logger} for this class, which is named + * "junit.framework.Test". + */ + + protected static final Logger log = Logger.getLogger + ( junit.framework.Test.class + ); + + /** + * This convenience method provides a version of {@link #fail( + * String Message )} that also excepts a {@link Throwable} cause + * for the test failure.<p> + */ + + static public void fail + ( String message, + Throwable t + ) + { + + AssertionFailedError ex = new AssertionFailedError + ( message + ); + + ex.initCause( t ); + + throw ex; + + } + + //************************************************************ + //************************************************************ + //************************************************************ + + static public void assertEquals( boolean[] expected, boolean[] actual ) + { + assertEquals( null, expected, actual ); + } + + /** + * Compares arrays of primitive values. + */ + + static public void assertEquals( String msg, boolean[] expected, boolean[] actual ) + { + + if( msg == null ) { + msg = ""; + } else { + msg = msg + " : "; + } + + if( expected == null && actual == null ) { + + return; + + } + + if( expected == null && actual != null ) { + + fail( msg+"Expected a null array." ); + + } + + if( expected != null && actual == null ) { + + fail( msg+"Not expecting a null array." ); + + } + + if (expected.length != actual.length) { + /* + * Only do message construction if we know that the assert will + * fail. + */ + assertEquals(msg + "length differs.", expected.length, + actual.length); + } + + for( int i=0; i<expected.length; i++ ) { + + if (expected[i] != actual[i]) { + /* + * Only do message construction if we know that the assert will + * fail. + */ + assertEquals(msg + "values differ: index=" + i, expected[i], + actual[i]); + } + + } + + } + + /** + * <p> + * Compares byte[]s by value (not reference). + * </p> + * <p> + * Note: This method will only be invoked if both arguments can be typed as + * byte[] by the compiler. If either argument is not strongly typed, you + * MUST case it to a byte[] or {@link #assertEquals(Object, Object)} will be + * invoked instead. + * </p> + * + * @param expected + * @param actual + */ + static public void assertEquals( byte[] expected, byte[] actual ) + { + + assertEquals( null, expected, actual ); + + } + + /** + * <p> + * Compares byte[]s by value (not reference). + * </p> + * <p> + * Note: This method will only be invoked if both arguments can be typed as + * byte[] by the compiler. If either argument is not strongly typed, you + * MUST case it to a byte[] or {@link #assertEquals(Object, Object)} will be + * invoked instead. + * </p> + * + * @param msg + * @param expected + * @param actual + */ + static public void assertEquals( String msg, byte[] expected, byte[] actual ) + { + + if( msg == null ) { + msg = ""; + } else { + msg = msg + " : "; + } + + if( expected == null && actual == null ) { + + return; + + } + + if( expected == null && actual != null ) { + + fail( msg+"Expected a null array." ); + + } + + if( expected != null && actual == null ) { + + fail( msg+"Not expecting a null array." ); + + } + + if (expected.length != actual.length) { + /* + * Only do message construction if we know that the assert will + * fail. + */ + assertEquals(msg + "length differs.", expected.length, + actual.length); + } + + for( int i=0; i<expected.length; i++ ) { + + if (expected[i] != actual[i]) { + /* + * Only do the message construction if we know that the assert + * will fail. + */ + assertEquals(msg + "values differ: index=" + i, expected[i], + actual[i]); + } + + } + + } + + static public void assertEquals( char[] expected, char[] actual ) + { + assertEquals( null, expected, actual ); + } + + static public void assertEquals( String msg, char[] expected, char[] actual ) + { + + if( msg == null ) { + msg = ""; + } else { + msg = msg + " : "; + } + + if( expected == null && actual == null ) { + + return; + + } + + if( expected == null && actual != null ) { + + fail( msg+"Expected a null array." ); + + } + + if( expected != null && actual == null ) { + + fail( msg+"Not expecting a null array." ); + + } + + if (expected.length != actual.length) { + /* + * Only do message construction if we know that the assert will + * fail. + */ + assertEquals(msg + "length differs.", expected.length, + actual.length); + } + + for( int i=0; i<expected.length; i++ ) { + + if (expected[i] != actual[i]) { + /* + * Only do the message construction if we know that the assert + * will fail. + */ + assertEquals(msg + "values differ: index=" + i, expected[i], + actual[i]); + } + + } + + } + + static public void assertEquals( short[] expected, short[] actual ) + { + assertEquals( null, expected, actual ); + } + + static public void assertEquals( String msg, short[] expected, short[] actual ) + { + + if( msg == null ) { + msg = ""; + } else { + msg = msg + " : "; + } + + if( expected == null && actual == null ) { + + return; + + } + + if( expected == null && actual != null ) { + + fail( msg+"Expected a null array." ); + + } + + if( expected != null && actual == null ) { + + fail( msg+"Not expecting a null array." ); + + } + + if (expected.length != actual.length) { + /* + * Only do message construction if we know that the assert will + * fail. + */ + assertEquals(msg + "length differs.", expected.length, + actual.length); + } + + for( int i=0; i<expected.length; i++ ) { + + if (expected[i] != actual[i]) { + /* + * Only do the message construction if we know that the assert + * will fail. + */ + assertEquals(msg + "values differ: index=" + i, expected[i], + actual[i]); + } + + } + + } + + static public void assertEquals( int[] expected, int[] actual ) + { + + assertEquals( null, expected, actual ); + + } + + static public void assertEquals( String msg, int[] expected, int[] actual ) + { + + if( msg == null ) { + msg = ""; + } else { + msg = msg + " : "; + } + + if( expected == null && actual == null ) { + + return; + + } + + if( expected == null && actual != null ) { + + fail( msg+"Expected a null array." ); + + } + + if( expected != null && actual == null ) { + + fail( msg+"Not expecting a null array." ); + + } + + if (expected.length != actual.length) { + /* + * Only do message construction if we know that the assert will + * fail. + */ + assertEquals(msg + "length differs.", expected.length, + actual.length); + } + + for( int i=0; i<expected.length; i++ ) { + + if (expected[i] != actual[i]) { + /* + * Only do the message construction if we know that the assert + * will fail. + */ + assertEquals(msg + "values differ: index=" + i, expected[i], + actual[i]); + } + + } + + } + + static public void assertEquals( long[] expected, long[] actual ) + { + assertEquals( null, expected, actual ); + } + + static public void assertEquals( String msg, long[] expected, long[] actual ) + { + + if( msg == null ) { + msg = ""; + } else { + msg = msg + " : "; + } + + if( expected == null && actual == null ) { + + return; + + } + + if( expected == null && actual != null ) { + + fail( msg+"Expected a null array." ); + + } + + if( expected != null && actual == null ) { + + fail( msg+"Not expecting a null array." ); + + } + + if (expected.length != actual.length) { + /* + * Only do message construction if we know that the assert will + * fail. + */ + assertEquals(msg + "length differs.", expected.length, + actual.length); + } + + for( int i=0; i<expected.length; i++ ) { + + if (expected[i] != actual[i]) { + /* + * Only do the message construction if we know that the assert + * will fail. + */ + assertEquals(msg + "values differ: index=" + i, expected[i], + actual[i]); + + } + + } + + } + + static public void assertEquals( float[] expected, float[] actual ) + { + assertEquals( null, expected, actual ); + } + + /** + * @todo Use smarter floating point comparison and offer parameter + * for controlling the floating point comparison. + */ + static public void assertEquals( String msg, float[] expected, float[] actual ) + { + + if( msg == null ) { + msg = ""; + } else { + msg = msg + " : "; + } + + if( expected == null && actual == null ) { + + return; + + } + + if( expected == null && actual != null ) { + + fail( msg+"Expected a null array." ); + + } + + if( expected != null && actual == null ) { + + fail( msg+"Not expecting a null array." ); + + } + + if (expected.length != actual.length) { + /* + * Only do message construction if we know that the assert will + * fail. + */ + assertEquals(msg + "length differs.", expected.length, + actual.length); + } + + for( int i=0; i<expected.length; i++ ) { + + float delta = 0f; + + try { + + assertEquals(expected[i], actual[i], delta); + + } catch (AssertionFailedError ex) { + + /* + * Only do the message construction once the assertion is known + * to fail. + */ + fail(msg + "values differ: index=" + i, ex); + + } + + } + + } + + static public void assertEquals( double[] expected, double[] actual ) + { + assertEquals( null, expected, actual ); + } + + /** + * @todo Use smarter floating point comparison and offer parameter + * for controlling the floating point comparison. + */ + static public void assertEquals( String msg, double[] expected, double[] actual ) + { + + if( msg == null ) { + msg = ""; + } else { + msg = msg + " : "; + } + + if( expected == null && actual == null ) { + + return; + + } + + if( expected == null && actual != null ) { + + fail( msg+"Expected a null array." ); + + } + + if( expected != null && actual == null ) { + + fail( msg+"Not expecting a null array." ); + + } + + if (expected.length != actual.length) { + /* + * Only do message construction if we know that the assert will + * fail. + */ + assertEquals(msg + "length differs.", expected.length, + actual.length); + } + + for( int i=0; i<expected.length; i++ ) { + + double delta = 0.0d; + + try { + + assertEquals(expected[i], actual[i], delta); + + } catch (AssertionFailedError ex) { + + /* + * Only do the message construction once the assertion is known + * to fail. + */ + fail(msg + "values differ: index=" + i, ex); + + } + + } + + } + + static public void assertEquals( Object[] expected, Object[] actual ) + { + assertEquals( null, expected, actual ); + } + + /** + * Compares arrays of {@link Object}s. The length of the arrays + * must agree, and each array element must agree. However the + * class of the arrays does NOT need to agree, e.g., an Object[] + * MAY compare as equals with a String[]. + */ + + static public void assertEquals( String msg, Object[] expected, Object[] actual ) + { + + if( msg == null ) { + msg = ""; + } else { + msg = msg + " : "; + } + + if( expected == null && actual == null ) { + + return; + + } + + if( expected == null && actual != null ) { + + fail( msg+"Expected a null array." ); + + } + + if( expected != null && actual == null ) { + + fail( msg+"Not expecting a null array." ); + + } + + if (expected.length != actual.length) { + /* + * Only do message construction if we know that the assert will + * fail. + */ + assertEquals(msg + "length differs.", expected.length, + actual.length); + } + + for( int i=0; i<expected.length; i++ ) { + + try { + + assertEquals(expected[i], actual[i]); + + } catch (AssertionFailedError ex) { + + /* + * Only do the message construction once the assertion is known + * to fail. + */ + + fail(msg + "values differ: index=" + i, ex); + + } + + } + + } + + //************************************************************ + //************************************************************ + //************************************************************ + + /** + * Test helper that can correctly compare arrays of primitives and + * arrays of objects as well as primitives and objects. + */ + static public void assertSameValue( Object expected, Object actual ) + { + assertSameValue( null, expected, actual ); + } + + /** + * Test helper that can correctly compare arrays of primitives and + * arrays of objects as well as primitives and objects. + * + * @todo This will throw a {@link ClassCastException} if + * <i>actual</i> is of a different primitive array type. Change + * the code to throw {@link AssertionFailedError} instead. + */ + static public void assertSameValue( String msg, Object expected, Object actual ) + { + + if( expected != null && expected.getClass().isArray() ) { + + if( expected.getClass().getComponentType().isPrimitive() ) { + + Class componentType = expected.getClass().getComponentType(); + + if( componentType.equals( Boolean.TYPE ) ) { + + assertEquals + ( msg, + (boolean[]) expected, + (boolean[]) actual + ); + + } else if( componentType.equals( Byte.TYPE ) ) { + + assertEquals + ( msg, + (byte[]) expected, + (byte[]) actual + ); + + } else if( componentType.equals( Character.TYPE ) ) { + + assertEquals + ( msg, + (char[]) expected, + (char[]) actual + ); + + } else if( componentType.equals( Short.TYPE ) ) { + + assertEquals + ( msg, + (short[]) expected, + (short[]) actual + ); + + } else if( componentType.equals( Integer.TYPE ) ) { + + assertEquals + ( msg, + (int[]) expected, + (int[]) actual + ); + + } else if( componentType.equals( Long.TYPE ) ) { + + assertEquals + ( msg, + (long[]) expected, + (long[]) actual + ); + + } else if( componentType.equals( Float.TYPE ) ) { + + assertEquals + ( msg, + (float[]) expected, + (float[]) actual + ); + + } else if( componentType.equals( Double.TYPE ) ) { + + assertEquals + ( msg, + (double[]) expected, + (double[]) actual + ); + + } else { + + throw new AssertionError(); + + } + + } else { + + assertTrue + ( msg, + java.util.Arrays.equals + ( (Object[]) expected, + (Object[]) actual + ) + ); + + } + + } else { + + assertEquals + ( msg, + expected, + actual + ); + + } + + } + + //************************************************************ + //************************************************************ + //************************************************************ + + /** + * Method verifies that the <i>actual</i> {@link Iterator} + * produces the expected objects in the expected order. Objects + * are compared using {@link Object#equals( Object other )}. Errors + * are reported if too few or too many objects are produced, etc. + */ + + static public void assertSameIterator + ( Object[] expected, + Iterator actual + ) + { + + assertSameIterator + ( "", + expected, + actual + ); + + } + + /** + * Method verifies that the <i>actual</i> {@link Iterator} + * produces the expected objects in the expected order. Objects + * are compared using {@link Object#equals( Object other )}. Errors + * are reported if too few or too many objects are produced, etc. + */ + + static public void assertSameIterator + ( String msg, + Object[] expected, + Iterator actual + ) + { + + int i = 0; + + while( actual.hasNext() ) { + + if( i >= expected.length ) { + + fail( msg+": The iterator is willing to visit more than "+ + expected.length+ + " objects." + ); + + } + + Object g = actual.next(); + +// if (!expected[i].equals(g)) { + try { + assertSameValue(expected[i],g); + } catch(AssertionFailedError ex) { + /* + * Only do message construction if we know that the assert will + * fail. + */ + fail(msg + ": Different objects at index=" + i + ": expected=" + + expected[i] + ", actual=" + g); + } + + i++; + + } + + if( i < expected.length ) { + + fail( msg+": The iterator SHOULD have visited "+expected.length+ + " objects, but only visited "+i+ + " objects." + ); + + } + + } + + /** + * Verifies that the iterator visits the specified objects in some + * arbitrary ordering and that the iterator is exhausted once all + * expected objects have been visited. The implementation uses a + * selection without replacement "pattern". + */ + + static public void assertSameIteratorAnyOrder + ( Comparable[] expected, + Iterator actual + ) + { + + assertSameIteratorAnyOrder + ( "", + expected, + actual + ); + + } + + /** + * Verifies that the iterator visits the specified objects in some + * arbitrary ordering and that the iterator is exhausted once all + * expected objects have been visited. The implementation uses a + * selection without replacement "pattern". + * + * FIXME Write test cases for this one. + * + * FIXME Can we we this without requiring the objects to implement + * {@link Comparable}? + */ + + static public void assertSameIteratorAnyOrder + ( String msg, + Comparable[] expected, + Iterator actual + ) + { + + // Populate a map that we will use to realize the match and + // selection without replacement logic. + + final int nrange = expected.length; + + java.util.Map range = new java.util.TreeMap(); + + for( int j=0; j<nrange; j++ ) { + + range.put + ( expected[ j ], + expected[ j ] + ); + + } + + // Do selection without replacement for the objects visited by + // iterator. + + for( int j=0; j<nrange; j++ ) { + + if( ! actual.hasNext() ) { + + fail( msg+": Index exhausted while expecting more object(s)"+ + ": index="+j + ); + + } + + Object actualObject = actual.next(); + + if( range.remove( actualObject ) == null ) { + + fail( "Object not expected"+ + ": index="+j+ + ", object="+actualObject + ); + + } + + } + + if( actual.hasNext() ) { + + fail( "Iterator will deliver too many objects." + ); + + } + + } + + //************************************************************ + //************************************************************ + //************************************************************ + + /** + * Test helper produces a random sequence of indices in the range [0:n-1] + * suitable for visiting the elements of an array of n elements in a random + * order. This is useful when you want to randomize the presentation of + * elements from two or more arrays. For example, known keys and values can + * be generated and their presentation order randomized by indexing with the + * returned array. + */ + + public static int[] getRandomOrder( final int n ) + { + + final class Pair + implements Comparable + { + public double r = Math.random(); + public int val; + public Pair( int val ) {this.val = val;} + public int compareTo(Object other) + { + if( this == other ) return 0; + if( this.r < ((Pair)other).r ) return -1; + else return 1; + } + + } + + Pair[] pairs = new Pair[ n ]; + + for( int i=0; i<n; i++ ) { + + pairs[ i ] = new Pair( i ); + + } + + java.util.Arrays.sort( pairs ); + + int order[] = new int[ n ]; + + for( int i=0; i<n; i++ ) { + + order[ i ] = pairs[ i ].val; + + } + + return order; + + } + + /** + * Random number generator used by {@link #getNormalInt( int range + * )}. + */ + + private Random m_random = new Random(); + + /** + * Returns a random integer normally distributed in [0:range]. + */ + + public int getNormalInt( int range ) + { + + final double bound = 3d; + + double rand = m_random.nextGaussian(); + + if( rand < -bound ) rand = -bound; + else if( rand > bound ) rand = bound; + + rand = ( rand + bound ) / ( 2 * bound ); + // normal distribution in [0:1]. + + if( rand < 0d ) rand = 0d; + // make sure it is not slightly + // negative + + int r = (int)( rand * range ); + + if( r > range ) { + + throw new AssertionError(); + + } + + return r; + + // @todo develop a test case for this method based on the code + // in comments below and migrate into TestCase2 (junit-ext + // package). + +// int[] r = new int[10000]; + +// for( int i=0; i<r.length; i++ ) { + +// r[ i ] = getNormalInt( len ); + +// } + +// java.util.Arrays.sort( r ); + + + } + + /** + * Returns a random but unique string of Unicode characters with a + * maximum length of len and a minimum length. + * + * @param len The maximum length of the string. Each generated + * literal will have a mean length of <code>len/2</code> and the + * lengths will be distributed using a normal distribution (bell + * curve). + * + * @param id A unique index used to obtain a unique string. + * Typically this is a one up identifier. + */ + + public String getRandomString( int len, int id ) + { + +// final String data = "0123456789!@#$%^&*()`~-_=+[{]}\\|;:'\",<.>/?QWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm"; + final String data = "0123456789QWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm"; + + int[] order = getRandomOrder( data.length() ); + + int n = getNormalInt( len ); + + StringBuffer sb = new StringBuffer( n ); + + int index = 0; + + for( int i=0; i<n; i++ ) { + + sb.append( data.charAt( order[ index++ ] ) ); + + if( index == order.length ) { + + index = 0; + + } + + } + + sb.append( id ); + + return sb.toString(); + + } + + /** + * Used to create objects of random type. + * + * @todo support generation of random array types and dates. + * + * @author thompsonbry + */ + protected class RandomType + { + + /** + * An explicit NULL property value. + */ + + final public static short NULL = 0; + +// /** +// * A directed link to another generic object in the same store. +// * +// * @see LinkValue +// */ +// +// final public static short LINK = 1; +// +// /** +// * A BLOB (metadata that supports access to very large binary +// * objects using a stream oriented API). +// */ +// +// final public static short BLOB = 2; + + /** + * The Java primitive type <em>boolean</em>. + * + * Note: Support for primitives recognizes the corresponding + * objects, e.g., {@link Boolean}, and special cases their + * serialization. This significantly reduces their the storage + * requirements. + */ + + final public static short BOOLEAN = 20; + final public static short BYTE = 21; + final public static short CHAR = 22; + final public static short SHORT = 23; + final public static short INT = 24; + final public static short LONG = 25; + final public static short FLOAT = 26; + final public static short DOUBLE = 27; + + /** + * Java Object (must implement {@link Serializable} or {@link + * Externalizable}). + */ + + final public static short OBJECT = 30; + + /** + * Array of Java objects (but not Java primitives). + */ + + final public static short OBJECT_ARRAY = 31; + + /** + * Array of Java primitives. + */ + + final public static short BOOLEAN_ARRAY = 40; + final public static short BYTE_ARRAY = 41; + final public static short CHAR_ARRAY = 42; + final public static short SHORT_ARRAY = 43; + final public static short INT_ARRAY = 44; + final public static short LONG_ARRAY = 45; + final public static short FLOAT_ARRAY = 46; + final public static short DOUBLE_ARRAY = 47; + + final public int[] randomType = new int[] { +// LINK, +// BLOB, + BOOLEAN, BYTE, CHAR, SHORT, INT, LONG, FLOAT, DOUBLE, + OBJECT, +// OBJECT_ARRAY, +// BOOLEAN_ARRAY, +// BYTE_ARRAY, +// CHAR_ARRAY, +// SHORT_ARRAY, +// INT_ARRAY, +// LONG_ARRAY, +// FLOAT_ARRAY, +// DOUBLE_ARRAY, + NULL // Note: null at end to make easy to exclude. + }; + + /** + * For random characters. + */ + static final private String alphabet = "0123456789QWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm"; + + /** + * Returns an object with a random type and random value. + * + * @param rnd The random# generator to use. + * + * @param allowNull When true, a null object reference may be + * returned. + * + * @return The random object. + */ + + public Object nextObject( Random rnd, boolean allowNull ) + { + + int[] types = randomType; + + int range = types.length; + + if( ! allowNull ) range--; // exclude null from types. + + int type = types[ rnd.nextInt( range ) ]; + + Object obj = null; + + switch( type ) { + + case NULL: { + return null; + } + case BOOLEAN: { + obj = ( rnd.nextBoolean() + ? Boolean.TRUE + : Boolean.FALSE + ); + break; + } + case BYTE: { + obj = new Byte( (byte) ( rnd.nextInt( 255 ) - 127 ) ); + break; + } + case CHAR: { + obj = new Character( alphabet.charAt( rnd.nextInt( alphabet.length() ) ) ); + break; + } + case SHORT: { + obj = new Short( (short) rnd.nextInt() ); + break; + } + case INT: { + obj = new Integer( rnd.nextInt() ); + break; + } + case LONG: { + obj = new Long( rnd.nextLong() ); + break; + } + case FLOAT: { + obj = new Float( rnd.nextFloat() ); + break; + } + case DOUBLE: { + obj = new Double( rnd.nextDouble() ); + break; + } + case OBJECT: { + obj = getRandomString( 40, rnd.nextInt() ); + break; + } +// OBJECT_ARRAY, +// BOOLEAN_ARRAY, +// BYTE_ARRAY, +// CHAR_ARRAY, +// SHORT_ARRAY, +// INT_ARRAY, +// LONG_ARRAY, +// FLOAT_ARRAY, +// DOUBLE_ARRAY + default: { + throw new AssertionError( "unknown type="+type ); + } + + } + + return obj; + + } + + } + + private RandomType _randomType = new RandomType(); + + /** + * Returns an object with a random type and random value. + * + * @param allowNull When true, a null object reference may be + * returned. + * + * @return The random object. + */ + + public Object getRandomObject( boolean allowNull ) + { + + return _randomType.nextObject( m_random, allowNull ); + + } + + /** + * Returns an object with a random type and random value. + * + * @param rnd The random generator. ... [truncated message content] |
From: <fko...@us...> - 2010-09-09 13:51:02
|
Revision: 3525 http://bigdata.svn.sourceforge.net/bigdata/?rev=3525&view=rev Author: fkoliver Date: 2010-09-09 13:50:53 +0000 (Thu, 09 Sep 2010) Log Message: ----------- Revert changes which brought sources from another project into this project. I apologize for making the change without proper consideration for license. Modified Paths: -------------- branches/maven_scaleout/bigdata-core/pom.xml branches/maven_scaleout/bigdata-core/src/test/deploy/testing/conf/standalone/ServerStarter.config branches/maven_scaleout/bigdata-core/thirdparty/maven.xml Added Paths: ----------- branches/maven_scaleout/bigdata-core/thirdparty/lib/cweb-junit-ext-1.1-b3-dev.jar Removed Paths: ------------- branches/maven_scaleout/bigdata-core/src/test/java/junit/ Property Changed: ---------------- branches/maven_scaleout/bigdata-core/ Property changes on: branches/maven_scaleout/bigdata-core ___________________________________________________________________ Deleted: svn:ignore - target Modified: branches/maven_scaleout/bigdata-core/pom.xml =================================================================== --- branches/maven_scaleout/bigdata-core/pom.xml 2010-09-08 20:52:07 UTC (rev 3524) +++ branches/maven_scaleout/bigdata-core/pom.xml 2010-09-09 13:50:53 UTC (rev 3525) @@ -216,6 +216,12 @@ </dependency> <dependency> <groupId>${thirdParty.groupId}</groupId> + <artifactId>cweb-junit-ext</artifactId> + <version>1.1.0-b3-dev</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${thirdParty.groupId}</groupId> <artifactId>cweb-commons</artifactId> <version>1.1.0-b2-dev</version> </dependency> Modified: branches/maven_scaleout/bigdata-core/src/test/deploy/testing/conf/standalone/ServerStarter.config =================================================================== --- branches/maven_scaleout/bigdata-core/src/test/deploy/testing/conf/standalone/ServerStarter.config 2010-09-08 20:52:07 UTC (rev 3524) +++ branches/maven_scaleout/bigdata-core/src/test/deploy/testing/conf/standalone/ServerStarter.config 2010-09-09 13:50:53 UTC (rev 3525) @@ -59,6 +59,7 @@ libdir+"icu4j-3_6.jar"+File.pathSeparator+ // test suites only! libdir+"junit-3.8.1.jar"+File.pathSeparator+ + libdir+"cweb-junit-ext-1.1-b2-dev.jar"+File.pathSeparator+ // main bigdata JAR. //libdir+ "bigdata-core.jar" Copied: branches/maven_scaleout/bigdata-core/thirdparty/lib/cweb-junit-ext-1.1-b3-dev.jar (from rev 3520, branches/maven_scaleout/bigdata-core/thirdparty/lib/cweb-junit-ext-1.1-b3-dev.jar) =================================================================== (Binary files differ) Modified: branches/maven_scaleout/bigdata-core/thirdparty/maven.xml =================================================================== --- branches/maven_scaleout/bigdata-core/thirdparty/maven.xml 2010-09-08 20:52:07 UTC (rev 3524) +++ branches/maven_scaleout/bigdata-core/thirdparty/maven.xml 2010-09-09 13:50:53 UTC (rev 3525) @@ -69,6 +69,7 @@ <installJar groupId="com.bigdata.thirdparty" artifactId="ctc-utils" version="5-4-2005" jar="${thirdparty.dir}/ctc_utils-5-4-2005.jar" /> <installJar groupId="com.bigdata.thirdparty" artifactId="cweb-commons" version="1.1.0-b2-dev" jar="${thirdparty.dir}/cweb-commons-1.1-b2-dev.jar" /> <installJar groupId="com.bigdata.thirdparty" artifactId="cweb-extser" version="0.1.0-b2-dev" jar="${thirdparty.dir}/cweb-extser-0.1-b2-dev.jar" /> + <installJar groupId="com.bigdata.thirdparty" artifactId="cweb-junit-ext" version="1.1.0-b3-dev" jar="${thirdparty.dir}/cweb-junit-ext-1.1-b3-dev.jar" /> <installJar groupId="com.bigdata.thirdparty" artifactId="dsi-utils" version="1.0.6-020610" jar="${thirdparty.dir}/dsi-utils-1.0.6-020610.jar" /> <installJar groupId="com.bigdata.thirdparty" artifactId="unimi-fastutil" version="5.1.5" jar="${thirdparty.dir}/fastutil-5.1.5.jar" /> <installJar groupId="com.bigdata.thirdparty" artifactId="high-scale-lib" version="1.1.2" jar="${thirdparty.dir}/high-scale-lib-v1.1.2.jar" /> This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <ble...@us...> - 2010-09-17 14:44:31
|
Revision: 3581 http://bigdata.svn.sourceforge.net/bigdata/?rev=3581&view=rev Author: blevine218 Date: 2010-09-17 14:44:23 +0000 (Fri, 17 Sep 2010) Log Message: ----------- changes paths in certain config files Modified Paths: -------------- branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/deploy/default-deploy.properties branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/jini/util/LookupStarter.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/config/ConfigDeployUtil.java branches/maven_scaleout/bigdata-core/src/test/deploy/testing/conf/bigdataStandaloneTesting.config branches/maven_scaleout/bigdata-core/src/test/deploy/testing/test.xml Property Changed: ---------------- branches/maven_scaleout/bigdata-core/ Property changes on: branches/maven_scaleout/bigdata-core ___________________________________________________________________ Added: svn:ignore + target .settings .classpath .project Modified: branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/deploy/default-deploy.properties =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/deploy/default-deploy.properties 2010-09-17 14:39:31 UTC (rev 3580) +++ branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/deploy/default-deploy.properties 2010-09-17 14:44:23 UTC (rev 3581) @@ -8,7 +8,7 @@ # Federation properties federation.name.description=Name of the federation to discover -federation.name.default=@FED@ +federation.name.default= federation.name.type=string federation.minNodes.description=Minimum number of nodes in the federation Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/jini/util/LookupStarter.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/jini/util/LookupStarter.java 2010-09-17 14:39:31 UTC (rev 3580) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/jini/util/LookupStarter.java 2010-09-17 14:44:23 UTC (rev 3581) @@ -25,6 +25,7 @@ package com.bigdata.service.jini.util; +import com.bigdata.util.config.ConfigDeployUtil; import com.bigdata.util.config.ConfigurationUtil; import com.bigdata.util.config.LogUtil; import com.bigdata.util.config.NicUtil; @@ -75,10 +76,10 @@ private static String defaultGroup = null; static { try { - thisHost = NicUtil.getIpAddress("default.nic", "default", true); - defaultGroup = - System.getProperty("federation.name", - "bigdata.test.group-"+thisHost); + thisHost = NicUtil.getIpAddress("default.nic", "default", false); + //defaultGroup = System.getProperty("federation.name","bigdata.test.group-"+thisHost); + defaultGroup = ConfigDeployUtil.getFederationName(); + } catch (Throwable t) { /* swallow */ } } private static String defaultCodebasePort = "23333"; Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/config/ConfigDeployUtil.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/config/ConfigDeployUtil.java 2010-09-17 14:39:31 UTC (rev 3580) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/config/ConfigDeployUtil.java 2010-09-17 14:44:23 UTC (rev 3581) @@ -55,6 +55,9 @@ private static final String MIN = ".min"; private static final String DESCRIPTION = ".description"; private static final String TYPE = ".type"; + + private static final String FALLBACK_FEDNAME_FORMAT = "bigdata.test.group-%s"; + private static final String TEMPLATE_TOKEN_PATTERN = "@.*@"; public static String getString(String parameter) throws ConfigurationException @@ -156,12 +159,49 @@ */ public static String[] getGroupsToDiscover() throws ConfigurationException { - String fedNameStr = System.getProperty("federation.name"); - if(fedNameStr == null) { - fedNameStr = getString("federation.name"); - } + String fedNameStr = getFederationName(); return fedNameStr.split(","); } + + /** + * Retrieve the federation name (also used as Jini group name) via this pecking pecking order: + * <ol> + * <li>From the Java system property: <code>federation.name</code></li> + * <li>From the deployment properties file. Note that a value from the deployment + * properties file that has not gone through token replacement is considered + * invalid. In this case, the next value in the pecking order is used.</li> + * <li>Using the fallback convention: <code>bigdata.test.group-<ipaddress></code></li> + * </ol> + * + * @return String the federation name + * + * @throws ConfigurationException + */ + public static String getFederationName() throws ConfigurationException + { + // If we have a system property override, use that. + String fedName = System.getProperty("federation.name"); + + // If not, look in the deploy properties + if(fedName == null) { + fedName = getString("federation.name"); + } + + // If not set in the deploy properties, then use the fallback name of + // "bigdata.test.group-<ipaddress>". This is primarily to support test + // environments and we should never get this far in production. + if (fedName == null || fedName.length() == 0) { + try { + String ipAddress = NicUtil.getIpAddress("default.nic", "default", true); + fedName = String.format(FALLBACK_FEDNAME_FORMAT, ipAddress); + } + catch (Exception e) { + throw new ConfigurationException("Error retrieving IP address while constructing fallback federation name.", e); + } + } + + return fedName; + } /** * Returns an array of <code>LookupLocator</code> instances that can Modified: branches/maven_scaleout/bigdata-core/src/test/deploy/testing/conf/bigdataStandaloneTesting.config =================================================================== --- branches/maven_scaleout/bigdata-core/src/test/deploy/testing/conf/bigdataStandaloneTesting.config 2010-09-17 14:39:31 UTC (rev 3580) +++ branches/maven_scaleout/bigdata-core/src/test/deploy/testing/conf/bigdataStandaloneTesting.config 2010-09-17 14:44:23 UTC (rev 3581) @@ -77,7 +77,8 @@ * MUST use unicast discovery and specify the federation name in * the [groups]. */ - static fedname = System.getProperty("federation.name","testBigdataStandalone"); + //static fedname = System.getProperty("federation.name","testBigdataStandalone"); + static fedname = ConfigDeployUtil.getFederationName(); static zrootname = System.getProperty("bigdata.zrootname","testBigdataStandaloneZroot"); private static appHome = System.getProperty("app.home", ConfigMath.getAbsolutePath(new File(".")) ); Modified: branches/maven_scaleout/bigdata-core/src/test/deploy/testing/test.xml =================================================================== --- branches/maven_scaleout/bigdata-core/src/test/deploy/testing/test.xml 2010-09-17 14:39:31 UTC (rev 3580) +++ branches/maven_scaleout/bigdata-core/src/test/deploy/testing/test.xml 2010-09-17 14:44:23 UTC (rev 3581) @@ -32,7 +32,7 @@ <target name="setup" > <exec executable="hostname" outputproperty="this.hostname" /> <property name="test.codebase" value="http://${this.hostname}:${test.codebase.port}/jsk-dl.jar" /> - <property name="federation.name" value="bigdata.test.group-${this.hostname}" /> + <!-- <property name="federation.name" value="bigdata.test.group-${this.hostname}" /> --> </target> <target name="junit" description="starts http class server, lookup service, runs junit tests, stops lookup service, stops http class server." @@ -100,7 +100,9 @@ <sysproperty key="log4j.configuration" value="${log4j.configuration}" /> <sysproperty key="codebase.port" value="${test.codebase.port}" /> <sysproperty key="java.net.preferIPv4Stack" value="${java.net.preferIPv4Stack}" /> + <!-- <sysproperty key="federation.name" value="${federation.name}" /> + --> <sysproperty key="default.nic" value="${default.nic}" /> </java> </target> @@ -120,7 +122,9 @@ <sysproperty key="java.security.policy" value="${java.security.policy}" /> <sysproperty key="log4j.configuration" value="${log4j.configuration}" /> <sysproperty key="java.net.preferIPv4Stack" value="${java.net.preferIPv4Stack}" /> + <!-- <sysproperty key="federation.name" value="${federation.name}" /> + --> <sysproperty key="default.nic" value="${default.nic}" /> <arg value="-stop" /> </java> This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <sgo...@us...> - 2010-09-20 23:46:21
|
Revision: 3601 http://bigdata.svn.sourceforge.net/bigdata/?rev=3601&view=rev Author: sgossard Date: 2010-09-20 23:46:12 +0000 (Mon, 20 Sep 2010) Log Message: ----------- [maven_scaleout] : Broke all direct dependency cycles with package 'com.bigdata'. Major change involves most calls to com.bigdata.LRUNexus and moving it into the 'com.bigdata.cache' package. This class was a singleton used for accessing a global cache, and was disabled by default. Future caching work should use the decorator pattern to avoid static knowledge of a cache. Modified Paths: -------------- branches/maven_scaleout/bigdata-core/bigdata-perf/bsbm/build.properties branches/maven_scaleout/bigdata-core/bigdata-perf/bsbm/src/resources/logging/log4j.properties branches/maven_scaleout/bigdata-core/bigdata-perf/btc/build.properties branches/maven_scaleout/bigdata-core/bigdata-perf/btc/src/resources/logging/log4j.properties branches/maven_scaleout/bigdata-core/bigdata-perf/lubm/build.properties branches/maven_scaleout/bigdata-core/bigdata-perf/uniprot/build.properties branches/maven_scaleout/bigdata-core/bigdata-perf/uniprot/src/resources/logging/log4j.properties branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster.config branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster16.config branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/Banner.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/BigdataStatics.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/AbstractBTree.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/IndexMetadata.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/IndexSegmentBuilder.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/IndexSegmentStore.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/Node.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/BCHMGlobalLRU.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/BCHMGlobalLRU2.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRU.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRURecycler.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRURecyclerExplicitDeleteRequired.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/IGlobalLRU.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/StoreAndAddressLRUCache.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/WeakReferenceGlobalLRU.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/counters/AbstractStatisticsCollector.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/journal/AbstractJournal.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/journal/TemporaryRawStore.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/journal/WORMStrategy.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/rawstore/AbstractRawStore.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/rawstore/IRawStore.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/rawstore/SimpleMemoryRawStore.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/resources/StoreManager.java branches/maven_scaleout/bigdata-core/src/test/deploy/testing/data/com/bigdata/cache/StressTestGlobalLRU.xml branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/btree/AbstractIndexSegmentTestCase.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/btree/TestAll_IndexSegment.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/btree/TestIndexSegmentBuilderWithCompactingMerge.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/btree/TestIndexSegmentBuilderWithIncrementalBuild.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/btree/TestIndexSegmentBuilderWithLargeTrees.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/btree/TestIndexSegmentBuilderWithSmallTree.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/btree/TestIndexSegmentWithBloomFilter.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/btree/TestNullValues.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/cache/StressTestGlobalLRU.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/cache/TestBCHMGlobalLRU.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/cache/TestBCHMGlobalLRU2WithStripedLocks.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/cache/TestBCHMGlobalLRU2WithStripedLocksAndLIRS.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/cache/TestBCHMGlobalLRU2WithThreadLocalBuffers.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/cache/TestBCHMGlobalLRU2WithThreadLocalBuffersAndLIRS.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/cache/TestBCHMGlobalLRUWithLIRS.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/journal/AbstractRestartSafeTestCase.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/journal/TestAbort.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/rawstore/SimpleFileRawStore.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/rdf/store/AbstractTestCase.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/service/AbstractEmbeddedFederationTestCase.java Added Paths: ----------- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/LRUNexus.java Removed Paths: ------------- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/LRUNexus.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/btree/TestIndexSegmentBuilderCacheInteraction.java Modified: branches/maven_scaleout/bigdata-core/bigdata-perf/bsbm/build.properties =================================================================== --- branches/maven_scaleout/bigdata-core/bigdata-perf/bsbm/build.properties 2010-09-20 21:37:58 UTC (rev 3600) +++ branches/maven_scaleout/bigdata-core/bigdata-perf/bsbm/build.properties 2010-09-20 23:46:12 UTC (rev 3601) @@ -182,13 +182,13 @@ # The record cache (empty for the default cache). #cache= -cache=-Dcom.bigdata.LRUNexus.enabled=false -#cache=-Dcom.bigdata.LRUNexus.class=com.bigdata.cache.BCHMGlobalLRU2 -Dcom.bigdata.LRUNexus.threadLocalBuffers=true -#cache=-Dcom.bigdata.LRUNexus.class=com.bigdata.cache.BCHMGlobalLRU -Dcom.bigdata.LRUNexus.accessPolicy=LIRS -Dcom.bigdata.LRUNexus.limitingCapacity=2000000 -#-Dcom.bigdata.LRUNexus.class=com.bigdata.cache.StoreAndAddressLRUCache -#-Dcom.bigdata.LRUNexus.class=com.bigdata.cache.HardReferenceGlobalLRURecycler +cache=-Dcom.bigdata.cache.LRUNexus.enabled=false +#cache=-Dcom.bigdata.cache.LRUNexus.class=com.bigdata.cache.BCHMGlobalLRU2 -Dcom.bigdata.cache.LRUNexus.threadLocalBuffers=true +#cache=-Dcom.bigdata.cache.LRUNexus.class=com.bigdata.cache.BCHMGlobalLRU -Dcom.bigdata.cache.LRUNexus.accessPolicy=LIRS -Dcom.bigdata.cache.LRUNexus.limitingCapacity=2000000 +#-Dcom.bigdata.cache.LRUNexus.class=com.bigdata.cache.StoreAndAddressLRUCache +#-Dcom.bigdata.cache.LRUNexus.class=com.bigdata.cache.HardReferenceGlobalLRURecycler # -## -Dcom.bigdata.LRUNexus.percentHeap=.1 +## -Dcom.bigdata.cache.LRUNexus.percentHeap=.1 # all jvm args for query. queryJvmArgs=-server -Xmx${bsbm.maxMem} ${gcopts} ${gcdebug} ${profiler} ${cache} -Dlog4j.configuration=log4j.properties Modified: branches/maven_scaleout/bigdata-core/bigdata-perf/bsbm/src/resources/logging/log4j.properties =================================================================== --- branches/maven_scaleout/bigdata-core/bigdata-perf/bsbm/src/resources/logging/log4j.properties 2010-09-20 21:37:58 UTC (rev 3600) +++ branches/maven_scaleout/bigdata-core/bigdata-perf/bsbm/src/resources/logging/log4j.properties 2010-09-20 23:46:12 UTC (rev 3601) @@ -13,7 +13,7 @@ # Note: logging here at INFO or DEBUG will significantly impact throughput! #log4j.logger.com.bigdata=INFO -log4j.logger.com.bigdata.LRUNexus=INFO +log4j.logger.com.bigdata.cache.LRUNexus=INFO #log4j.logger.com.bigdata.rdf.sail.BigdataSail=INFO #log4j.logger.com.bigdata.rdf.sail.BigdataEvaluationStrategyImpl2=INFO Modified: branches/maven_scaleout/bigdata-core/bigdata-perf/btc/build.properties =================================================================== --- branches/maven_scaleout/bigdata-core/bigdata-perf/btc/build.properties 2010-09-20 21:37:58 UTC (rev 3600) +++ branches/maven_scaleout/bigdata-core/bigdata-perf/btc/build.properties 2010-09-20 23:46:12 UTC (rev 3601) @@ -61,11 +61,11 @@ # The record cache (empty for the default cache). #cache= -cache=-Dcom.bigdata.LRUNexus.enabled=false -#cache=-Dcom.bigdata.LRUNexus.class=com.bigdata.cache.BCHMGlobalLRU2 -Dcom.bigdata.LRUNexus.threadLocalBuffers=true -#cache=-Dcom.bigdata.LRUNexus.class=com.bigdata.cache.BCHMGlobalLRU -Dcom.bigdata.LRUNexus.accessPolicy=LIRS -Dcom.bigdata.LRUNexus.limitingCapacity=2000000 -#-Dcom.bigdata.LRUNexus.class=com.bigdata.cache.StoreAndAddressLRUCache -#-Dcom.bigdata.LRUNexus.class=com.bigdata.cache.HardReferenceGlobalLRURecycler +cache=-Dcom.bigdata.cache.LRUNexus.enabled=false +#cache=-Dcom.bigdata.cache.LRUNexus.class=com.bigdata.cache.BCHMGlobalLRU2 -Dcom.bigdata.cache.LRUNexus.threadLocalBuffers=true +#cache=-Dcom.bigdata.cache.LRUNexus.class=com.bigdata.cache.BCHMGlobalLRU -Dcom.bigdata.cache.LRUNexus.accessPolicy=LIRS -Dcom.bigdata.cache.LRUNexus.limitingCapacity=2000000 +#-Dcom.bigdata.cache.LRUNexus.class=com.bigdata.cache.StoreAndAddressLRUCache +#-Dcom.bigdata.cache.LRUNexus.class=com.bigdata.cache.HardReferenceGlobalLRURecycler # all jvm args for query. queryJvmArgs=-server -Xmx${maxMem} ${gcopts} ${gcdebug} ${profiler} ${cache} -Dlog4j.configuration=file:src/resources/logging/log4j.properties Modified: branches/maven_scaleout/bigdata-core/bigdata-perf/btc/src/resources/logging/log4j.properties =================================================================== --- branches/maven_scaleout/bigdata-core/bigdata-perf/btc/src/resources/logging/log4j.properties 2010-09-20 21:37:58 UTC (rev 3600) +++ branches/maven_scaleout/bigdata-core/bigdata-perf/btc/src/resources/logging/log4j.properties 2010-09-20 23:46:12 UTC (rev 3601) @@ -13,7 +13,7 @@ # Note: logging here at INFO or DEBUG will significantly impact throughput! #log4j.logger.com.bigdata=INFO -log4j.logger.com.bigdata.LRUNexus=INFO +log4j.logger.com.bigdata.cache.LRUNexus=INFO #log4j.logger.com.bigdata.rdf.sail.BigdataSail=INFO #log4j.logger.com.bigdata.rdf.sail.BigdataEvaluationStrategyImpl2=INFO Modified: branches/maven_scaleout/bigdata-core/bigdata-perf/lubm/build.properties =================================================================== --- branches/maven_scaleout/bigdata-core/bigdata-perf/lubm/build.properties 2010-09-20 21:37:58 UTC (rev 3600) +++ branches/maven_scaleout/bigdata-core/bigdata-perf/lubm/build.properties 2010-09-20 23:46:12 UTC (rev 3601) @@ -129,11 +129,11 @@ # The record cache (empty for the default cache). #cache= -cache=-Dcom.bigdata.LRUNexus.enabled=false -#cache=-Dcom.bigdata.LRUNexus.class=com.bigdata.cache.BCHMGlobalLRU2 -Dcom.bigdata.LRUNexus.threadLocalBuffers=true -#cache=-Dcom.bigdata.LRUNexus.class=com.bigdata.cache.BCHMGlobalLRU -Dcom.bigdata.LRUNexus.accessPolicy=LIRS -Dcom.bigdata.LRUNexus.limitingCapacity=2000000 -#-Dcom.bigdata.LRUNexus.class=com.bigdata.cache.StoreAndAddressLRUCache -#-Dcom.bigdata.LRUNexus.class=com.bigdata.cache.HardReferenceGlobalLRURecycler +cache=-Dcom.bigdata.cache.LRUNexus.enabled=false +#cache=-Dcom.bigdata.cache.LRUNexus.class=com.bigdata.cache.BCHMGlobalLRU2 -Dcom.bigdata.cache.LRUNexus.threadLocalBuffers=true +#cache=-Dcom.bigdata.cache.LRUNexus.class=com.bigdata.cache.BCHMGlobalLRU -Dcom.bigdata.cache.LRUNexus.accessPolicy=LIRS -Dcom.bigdata.cache.LRUNexus.limitingCapacity=2000000 +#-Dcom.bigdata.cache.LRUNexus.class=com.bigdata.cache.StoreAndAddressLRUCache +#-Dcom.bigdata.cache.LRUNexus.class=com.bigdata.cache.HardReferenceGlobalLRURecycler # all jvm args for query. queryJvmArgs=-server -Xmx${lubm.maxMem} ${gcopts} ${gcdebug} ${profiler} ${cache} -Dlog4j.configuration=log4j.properties Modified: branches/maven_scaleout/bigdata-core/bigdata-perf/uniprot/build.properties =================================================================== --- branches/maven_scaleout/bigdata-core/bigdata-perf/uniprot/build.properties 2010-09-20 21:37:58 UTC (rev 3600) +++ branches/maven_scaleout/bigdata-core/bigdata-perf/uniprot/build.properties 2010-09-20 23:46:12 UTC (rev 3601) @@ -60,11 +60,11 @@ # The record cache (empty for the default cache). #cache= -cache=-Dcom.bigdata.LRUNexus.enabled=false -#cache=-Dcom.bigdata.LRUNexus.class=com.bigdata.cache.BCHMGlobalLRU2 -Dcom.bigdata.LRUNexus.threadLocalBuffers=true -#cache=-Dcom.bigdata.LRUNexus.class=com.bigdata.cache.BCHMGlobalLRU -Dcom.bigdata.LRUNexus.accessPolicy=LIRS -Dcom.bigdata.LRUNexus.limitingCapacity=2000000 -#-Dcom.bigdata.LRUNexus.class=com.bigdata.cache.StoreAndAddressLRUCache -#-Dcom.bigdata.LRUNexus.class=com.bigdata.cache.HardReferenceGlobalLRURecycler +cache=-Dcom.bigdata.cache.LRUNexus.enabled=false +#cache=-Dcom.bigdata.cache.LRUNexus.class=com.bigdata.cache.BCHMGlobalLRU2 -Dcom.bigdata.cache.LRUNexus.threadLocalBuffers=true +#cache=-Dcom.bigdata.cache.LRUNexus.class=com.bigdata.cache.BCHMGlobalLRU -Dcom.bigdata.cache.LRUNexus.accessPolicy=LIRS -Dcom.bigdata.cache.LRUNexus.limitingCapacity=2000000 +#-Dcom.bigdata.cache.LRUNexus.class=com.bigdata.cache.StoreAndAddressLRUCache +#-Dcom.bigdata.cache.LRUNexus.class=com.bigdata.cache.HardReferenceGlobalLRURecycler # all jvm args for query. queryJvmArgs=-server -Xmx${maxMem} ${gcopts} ${gcdebug} ${profiler} ${cache} -Dlog4j.configuration=file:src/resources/logging/log4j.properties Modified: branches/maven_scaleout/bigdata-core/bigdata-perf/uniprot/src/resources/logging/log4j.properties =================================================================== --- branches/maven_scaleout/bigdata-core/bigdata-perf/uniprot/src/resources/logging/log4j.properties 2010-09-20 21:37:58 UTC (rev 3600) +++ branches/maven_scaleout/bigdata-core/bigdata-perf/uniprot/src/resources/logging/log4j.properties 2010-09-20 23:46:12 UTC (rev 3601) @@ -13,7 +13,7 @@ # Note: logging here at INFO or DEBUG will significantly impact throughput! #log4j.logger.com.bigdata=INFO -log4j.logger.com.bigdata.LRUNexus=INFO +log4j.logger.com.bigdata.cache.LRUNexus=INFO #log4j.logger.com.bigdata.rdf.sail.BigdataSail=INFO #log4j.logger.com.bigdata.rdf.sail.BigdataEvaluationStrategyImpl2=INFO Modified: branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster.config =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster.config 2010-09-20 21:37:58 UTC (rev 3600) +++ branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster.config 2010-09-20 23:46:12 UTC (rev 3601) @@ -629,15 +629,15 @@ "jini", "org.apache.zookeeper.server.quorum.QuorumPeerMain", -//BTM "com.bigdata.service.jini.TransactionServer", -//BTM "com.bigdata.service.jini.MetadataServer", + "com.bigdata.service.jini.TransactionServer", + "com.bigdata.service.jini.MetadataServer", "com.bigdata.service.jini.DataServer", -//BTM "com.bigdata.service.jini.LoadBalancerServer", + "com.bigdata.service.jini.LoadBalancerServer", "com.bigdata.service.jini.ClientServer", -"com.bigdata.transaction.ServiceImpl", -"com.bigdata.metadata.ServiceImpl", -"com.bigdata.loadbalancer.ServiceImpl" +//BTM "com.bigdata.transaction.ServiceImpl", +//BTM "com.bigdata.metadata.ServiceImpl", +//BTM "com.bigdata.loadbalancer.ServiceImpl" }; Modified: branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster16.config =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster16.config 2010-09-20 21:37:58 UTC (rev 3600) +++ branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster16.config 2010-09-20 23:46:12 UTC (rev 3601) @@ -322,15 +322,15 @@ /* * Override the LRU buffer capacity. * - * See com.bigdata.LRUNexus.Options for configuration info. Note that if + * See com.bigdata.cache.LRUNexus.Options for configuration info. Note that if * you disable the LRUNexus you will loose the leaf cache for the index * segments, which is a big penalty. - //"-Dcom.bigdata.LRUNexus.enabled=false", + //"-Dcom.bigdata.cache.LRUNexus.enabled=false", // option may be used to select the higher throughput impl. - "-Dcom.bigdata.LRUNexus.class=com.bigdata.cache.HardReferenceGlobalLRURecyclerExplicitDeleteRequired", + "-Dcom.bigdata.cache.LRUNexus.class=com.bigdata.cache.HardReferenceGlobalLRURecyclerExplicitDeleteRequired", // option may be used to allocate more heap to the LRUNexus. - "-Dcom.bigdata.LRUNexus.percentHeap=.2", - "-Dcom.bigdata.LRUNexus.indexSegmentBuildPopulatesCache=true", // default true + "-Dcom.bigdata.cache.LRUNexus.percentHeap=.2", + "-Dcom.bigdata.cache.LRUNexus.indexSegmentBuildPopulatesCache=true", // default true */ }; Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/Banner.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/Banner.java 2010-09-20 21:37:58 UTC (rev 3600) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/Banner.java 2010-09-20 23:46:12 UTC (rev 3601) @@ -31,12 +31,12 @@ import java.lang.reflect.Method; import java.util.Date; +import com.bigdata.util.config.ConfigDeployUtil; +import com.bigdata.util.config.NicUtil; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.system.SystemUtil; -import com.bigdata.counters.AbstractStatisticsCollector; - /** * Class has a static method which writes a copyright banner on stdout once per * JVM. This method is invoked from several core classes in order to ensure that @@ -48,6 +48,17 @@ public class Banner { private static boolean didBanner; + private static final String HOSTNAME; + static { + String val; + try { + val = NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false); + } catch(Throwable t) {//for now, maintain same failure logic as used previously + t.printStackTrace(); + val = NicUtil.getIpAddressByLocalHost(); + } + HOSTNAME = val; + } /** * Environment variables understood by the {@link Banner} class. @@ -153,7 +164,7 @@ "\n"+// "\nCopyright SYSTAP, LLC 2006-2010. All rights reserved."+// "\n"+// - "\n"+AbstractStatisticsCollector.fullyQualifiedHostName+// + "\n"+HOSTNAME+// "\n"+new Date()+// "\n"+SystemUtil.operatingSystem() + "/" + SystemUtil.osVersion() + " " + SystemUtil.architecture() + // Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/BigdataStatics.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/BigdataStatics.java 2010-09-20 21:37:58 UTC (rev 3600) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/BigdataStatics.java 2010-09-20 23:46:12 UTC (rev 3601) @@ -27,7 +27,6 @@ package com.bigdata; -import com.bigdata.jini.start.process.ProcessHelper; /** * A class for those few statics that it makes sense to reference from other @@ -50,9 +49,9 @@ * {@link System#out} when that child process is executed. This makes it * easy to track down why a child process dies during service start. If you * want to see more output from the child process, then you should set the - * log level for the {@link ProcessHelper} class to INFO. + * log level for the {@link com.bigdata.jini.start.process.ProcessHelper} class to INFO. * - * @see ProcessHelper + * @see com.bigdata.jini.start.process.ProcessHelper */ public static int echoProcessStartupLineCount = 20; Deleted: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/LRUNexus.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/LRUNexus.java 2010-09-20 21:37:58 UTC (rev 3600) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/LRUNexus.java 2010-09-20 23:46:12 UTC (rev 3601) @@ -1,950 +0,0 @@ -/* - -Copyright (C) SYSTAP, LLC 2006-2008. All rights reserved. - -Contact: - SYSTAP, LLC - 4501 Tower Road - Greensboro, NC 27410 - lic...@bi... - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ -/* - * Created on Sep 8, 2009 - */ - -package com.bigdata; - -import java.lang.management.ManagementFactory; -import java.lang.management.MemoryPoolMXBean; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.util.Properties; - -import org.apache.log4j.Logger; - -import com.bigdata.io.BytesUtil; -import com.bigdata.btree.IndexMetadata; -import com.bigdata.btree.IndexSegment; -import com.bigdata.btree.IndexSegmentBuilder; -import com.bigdata.cache.BCHMGlobalLRU; -import com.bigdata.cache.BCHMGlobalLRU2; -import com.bigdata.cache.HardReferenceGlobalLRU; -import com.bigdata.cache.HardReferenceGlobalLRURecycler; -import com.bigdata.cache.HardReferenceGlobalLRURecyclerExplicitDeleteRequired; -import com.bigdata.cache.IGlobalLRU; -import com.bigdata.cache.WeakReferenceGlobalLRU; -import com.bigdata.cache.IGlobalLRU.ILRUCache; -import com.bigdata.journal.AbstractJournal; -import com.bigdata.journal.IBufferStrategy; -import com.bigdata.journal.TemporaryRawStore; -import com.bigdata.rawstore.AbstractRawStore; -import com.bigdata.rawstore.Bytes; -import com.bigdata.rawstore.IAddressManager; -import com.bigdata.rawstore.IRawStore; -import com.bigdata.rawstore.WormAddressManager; - -/** - * Static singleton factory used to configure the record level cache behavior - * for bigdata within the current JVM. The configuration is specified using - * system properties defined by {@link Options}. - * - * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ - * - * FIXME LRUNexus : writes MUST must be "isolated" until the commit. - * Isolated indices MUST have their own cache backed by the shared LRU - * (actually, they are on the shared temporary store so that helps). - * Unisolated indices SHOULD have their own cache backed by the shared - * LRU. At commit, any records in the "isolated" cache for a B+Tree - * should be putAll() onto the unisolated cache for the backing store. - * This way, we do not need to do anything if there is an abort(). - * <p> - * There are two quick fixes: (1) Disable the Global LRU; and (2) - * discard the cache if there is an abort on a store. The latter is - * pretty easy since we only have one store with abort semantics, which - * is the {@link AbstractJournal}, so that is how this is being handled - * right now by {@link AbstractJournal#abort()}. - * <p> - * An optimization would essentially isolate the writes on the cache - * per BTree or between commits. At the commit point, the written - * records would be migrated into the "committed" cache for the store. - * The caller would read on the uncommitted cache, which would read - * through to the "committed" cache. This would prevent incorrect reads - * without requiring us to throw away valid records in the cache. This - * could be a significant performance gain if aborts are common on a - * machine with a lot of RAM. - * - * @todo Test w/ G1 <code>-XX:+UnlockExperimentalVMOptions -XX:+UseG1GC</code> - * <p> - * G1 appears faster for query, but somewhat slower for load. This is - * probably related to the increased memory demand during load (more of - * the data winds up buffered). G1 might work for both use cases with a - * smaller portion of the heap given over to buffers. - * <p> - * G1 can also trip a crash, at least during load. There is a Sun incident - * ID# 1609804 for this. - * - * @todo Look into the memory pool threshold notification mechanism. See - * {@link ManagementFactory#getMemoryPoolMXBeans()} and - * {@link MemoryPoolMXBean}. TonyP suggests that tracking the old - * generation occupancy may be a better metric (more stable). The tricky - * part is to identify which pool(s?) correspond(s) to the old generation. - * Once that is done, the idea is to set a notification threshold using - * {@link MemoryPoolMXBean#setUsageThreshold(long)} and to only clear - * references from the tail of the global LRU when we have exceeded that - * threshold. Reading the javadoc, it seems that threshold notification - * would probably come after a (full) GC. The goal would have to be - * something like reducing the bytesInMemory to some percentage of its - * value at threshold notification (e.g., 80%). Since we can't directly - * control that and the feedback from the JVM is only at full GC - * intervals, we need to simply discard some percentage of the references - * from the tail of the global LRU. We could actually adjust the desired - * #of references on the LRU if that metric appears to be relatively - * stable. However, note that the average #of bytes per reference and the - * average #of instances of a reference on the LRU are not necessarily - * stable values. We could also examine the recordCount (total cache size - * across all caches). If weak references are cleared on an ongoing basis - * rather than during the full GC mark phase, then that will be very close - * to the real hard reference count. - * - * @todo Does it make sense to both buffer the index segment nodes region and - * buffer the nodes and leaves? [buffering the nodes region is an option.] - * - * @todo Note that a r/w store will require an approach in which addresses are - * PURGED from the store's cache during the commit protocol. That might be - * handled at the tx layer. - * - * @todo Better ergonomics! Perhaps keep some minimum amount for the JVM and - * then set a trigger on the GC time and if it crosses 5-10% of the CPU - * time for the application, then reduce the maximum bytes allowed for the - * global LRU buffer. - * - * @see Options - */ -public class LRUNexus { - - protected static final transient Logger log = Logger - .getLogger(LRUNexus.class); - - /** - * These options are MUST BE specified as <em>ENVIRONMENT</em> variables on - * the command line when you start the JVM. The options control the - * existence of and behavior of the {@link LRUNexus#INSTANCE}. - * - * @author <a href="mailto:tho...@us...">Bryan - * Thompson</a> - * @version $Id$ - */ - public static interface Options { - - /** - * Option may be used to (en|dis)able the {@link LRUNexus#INSTANCE} - * (default {@value #DEFAULT_ENABLED}). - */ - String ENABLED = LRUNexus.class.getName() + ".enabled"; - - /** - * Note: The {@link LRUNexus} is now disabled by default. With the - * removal of the synchronization bottlenecks in the low levels of the - * RW and WORM Journal modes, it is now more efficient to NOT use the - * embedded concurrent non-blocking caches and to pass through the - * burden of cache operations to the OS. Also, due to the large number - * of records retained by the embedded cache and the length of time that - * those records are retained, the embedded concurrent non-blocking - * caches have proven to be a challenge to the garbage collector. If - * enabled, it is advisable to use the - * <code>-XX:+UseParallelOldGC</code> GC option in order to avoid JVM - * hangs or crashes. - */ - String DEFAULT_ENABLED = "false"; - - /** - * The maximum heap capacity as a percentage of the JVM heap expressed - * as a value in <code>[0.0:1.0]</code>. This is used IFF - * {@link #MAX_HEAP} is not specified or is ZERO (0), which is its - * default value. If both options are zero, then the maximum heap is - * understood to be zero and the {@link LRUNexus#INSTANCE} will be - * disabled. - */ - String PERCENT_HEAP = LRUNexus.class.getName() + ".percentHeap"; - - /** - * The default ({@value #DEFAULT_PERCENT_HEAP}) is a bit conservative. - * It is designed to leave some room for application data objects and - * GC. You may be able to get away with significantly more on machines - * with large RAM, or just specify the buffer heap size directly using - * {@link #MAX_HEAP}. - */ - String DEFAULT_PERCENT_HEAP = ".1"; - - /** - * This option overrides {@link #PERCENT_HEAP} and directly specifies - * the maximum capacity of the {@link LRUNexus#INSTANCE} in bytes. If - * both options are zero, then the maximum heap is understood to be zero - * and the {@link LRUNexus#INSTANCE} will be disabled. Legal examples - * include: - * - * <pre> - * 30000000 - * 400m - * 2Gb - * </pre> - * - * @see BytesUtil#getByteCount(String) - */ - String MAX_HEAP = LRUNexus.class.getName() + ".maxHeap"; - - String DEFAULT_MAX_HEAP = "0"; - - /** - * The percent of the maximum bytes which the LRU may buffer to be - * cleared from the LRU when evicting the LRU entry (default - * {@value #DEFAULT_PERCENT_CLEARED}). This parameter provides some - * "batching" of evictions but is not used by all {@link IGlobalLRU} - * implementations. - */ - String PERCENT_CLEARED = LRUNexus.class.getName() + ".percentCleared"; - - String DEFAULT_PERCENT_CLEARED = ".01"; - - /** - * The name of {@link IGlobalLRU} implementation class. - * - * @see #DEFAULT_CLASS - */ - String CLASS = LRUNexus.class.getName() + ".class"; - - /** - * The default {@link IGlobalLRU} implementation class ( - * {@value #DEFAULT_CLASS}). - * - * FIXME The {@link HardReferenceGlobalLRURecycler} has less throughput - * than the {@link HardReferenceGlobalLRU} but I want to test the - * {@link HardReferenceGlobalLRU} more throughly on high throughput - * cluster data loads to make sure that it is performing correctly. - * <p> - * Note: It is also possible that the {@link HardReferenceGlobalLRU} - * causes problems with the tenured generation since the Entry instances - * are always new, but they could last quite a while before eviction - * from the LRU position if there is a large heap. - * <p> - * Scale-out should use the - * {@link HardReferenceGlobalLRURecyclerExplicitDeleteRequired} to avoid - * giving away the cached index segment records when an index segment - * store is closed by a timeout. - * <p> - * For scale-up, the {@link HardReferenceGlobalLRURecycler} is fine. We - * are not re-opening stores all the time so the weak value reference - * semantics of that class do not cause a problem. - * <p> - * The recently written {@link BCHMGlobalLRU2} implementation should be - * ideal for both scale-out and scale-up once it has been tested more - * throughly. Even better would be a LIRS access policy for that class. - */ - String DEFAULT_CLASS = HardReferenceGlobalLRURecycler.class.getName(); -// String DEFAULT_CLASS = BCHMGlobalLRU2.class.getName(); - - /** - * The load factor for the cache instances. - */ - String LOAD_FACTOR = LRUNexus.class.getName() + ".loadFactor"; - - String DEFAULT_LOAD_FACTOR = ".75"; - - /** - * The concurrency level for the backing hash map(s). This property is - * not understood by all implementations. A value of ZERO (0) is - * interpreted in a special manner by {@link BCHMGlobalLRU2}. - */ - String CONCURRENCY_LEVEL = LRUNexus.class.getName() - + ".concurrencyLevel"; - - String DEFAULT_CONCURRENCY_LEVEL = "16"; - - /** - * When <code>true</code> the cache will use true <em>per-thread</em> - * buffers to absorb access policy updates. When <code>false</code>, the - * cache will use striped locks protecting a fixed array of buffers. - * This property is not understood by all implementations. - * */ - String THREAD_LOCAL_BUFFERS = LRUNexus.class.getName() - + ".threadLocalBuffers"; - - String DEFAULT_THREAD_LOCAL_BUFFERS = "false"; - - /** - * The initial capacity for the cache instances. - */ - String INITIAL_CAPACITY = LRUNexus.class.getName() + ".initialCapacity"; - - String DEFAULT_INITIAL_CAPACITY = "16"; - - /** - * The limiting cache capacity across all cache instances. - * - * @see #DEFAULT_LIMITING_CAPACITY - * - * @deprecated At the moment, this option is only understood by the - * {@link BCHMGlobalLRU}. The option may or may not survive - * as that feature is alpha. - */ - String LIMITING_CAPACITY = LRUNexus.class.getName()+".limitingCapacity"; - - String DEFAULT_LIMITING_CAPACITY = "" + (2 * Bytes.megabyte); - - /** - * The capacity of the thread-local buffer used to amortize the cost of - * updating the access policy. This option is only understood by select - * {@link IGlobalLRU} implementations. - */ - String THREAD_LOCAL_BUFFER_CAPACITY = LRUNexus.class.getName() - + ".threadLocalBufferCapacity"; - - String DEFAULT_THREAD_LOCAL_BUFFER_CAPACITY = "128"; - - /** - * The access policy (LIRS, LRU, etc). At the moment, this option is - * only understood by the {@link BCHMGlobalLRU}. - * - * @see #DEFAULT_ACCESS_POLICY_ENUM - */ - String ACCESS_POLICY = LRUNexus.class.getName() + ".accessPolicy"; - - String DEFAULT_ACCESS_POLICY = AccessPolicyEnum.LRU.toString(); - - /** - * The minimum #of per-{@link IRawStore} cache instances that will be - * retained by hard references when using an {@link IGlobalLRU} based on - * a weak value hash map such as {@link WeakReferenceGlobalLRU}. This - * controls the size of a hard reference ring buffer backing a weak - * value hash map. The actual number of cache instances will be less if - * fewer stores have been opened or if open stores have been - * {@link IRawStore#deleteResources() destroyed}. More cache instances - * will exist if there are hard references to more {@link IRawStore} - * instances. - */ - String MIN_CACHE_SET_SIZE = LRUNexus.class.getName()+".minCacheSetSize"; - - String DEFAULT_MIN_CACHE_SET_SIZE = "5"; - - /** - * When <code>true</code>, the {@link IndexSegmentBuilder} will - * pre-populate the {@link IGlobalLRU} cache with the nodes and leaves - * of the new index segment during the build or merge operation (default - * {@value #DEFAULT_INDEX_SEGMENT_BUILD_POPULATES_CACHE}). - */ - String INDEX_SEGMENT_BUILD_POPULATES_CACHE = LRUNexus.class.getName() - + ".indexSegmentBuildPopulatesCache"; - - /** - * FIXME Verify that [true] is the right behavior for scale-out. This is - * being tested in combination with the - * {@link HardReferenceGlobalLRURecyclerExplicitDeleteRequired}. - */ - String DEFAULT_INDEX_SEGMENT_BUILD_POPULATES_CACHE = "true"; - - } - - - /** - * Global instance. - * <p> - * Note: A <a href="http://bugs.sun.com/view_bug.do?bug_id=6880903">Sun G1 - * bug in JDK 1.6.0_16</a> provides a false estimate of the available - * memory. - * - * @see Options - */ - public static final IGlobalLRU<Long, Object> INSTANCE; - - /** - * The access policy. Not all {@link IGlobalLRU} implementations support - * multiple access policies. Check the specific implementation to see which - * policies it supports. - * - * @author <a href="mailto:tho...@us...">Bryan - * Thompson</a> - * @version $Id$ - */ - public static enum AccessPolicyEnum { - - /** - * Least-recently used access policy. - */ - LRU, - - /** - * An access policy designed to avoid displacement of frequently used - * cache entries by scans of infrequently used items. - * - * @see <a - * href="http://portal.acm.org/citation.cfm?doid=511334.511340">LIRS: - * an efficient low inter-reference recency set replacement policy - * to improve buffer cache performance</a> and <a - * href="http://www.ece.eng.wayne.edu/~sjiang/Projects/LIRS/sig02.ppt" - * >LIRS : An Efficient Replacement Policy to Improve Buffer Cache - * Performance.</a> - */ - LIRS; - - private AccessPolicyEnum() { - } - - } - - /** - * A class which reflects the configuration {@link Options}. - * - * @author <a href="mailto:tho...@us...">Bryan - * Thompson</a> - * @version $Id$ - */ - public static class CacheSettings { - - /** - * @see Options#ENABLED - */ - public final boolean enabled; - - /** - * @see Options#INDEX_SEGMENT_BUILD_POPULATES_CACHE - */ - public final boolean indexSegmentBuildPopulatesCache; - - /** - * The {@link IGlobalLRU} implementation class. - * - * @see Options#CLASS - */ - public final Class<? extends IGlobalLRU> cls; - - /** - * The load factor for the backing hash map(s). - * - * @see Options#LOAD_FACTOR - */ - public final float loadFactor; - - /** - * The concurrency level for the backing hash map(s). This property is - * not understood by all implementations. - * - * @see Options#CONCURRENCY_LEVEL - */ - public final int concurrencyLevel; - - /** - * When <code>true</code> the cache will use true <em>per-thread</em> - * buffers to absorb access policy updates. When <code>false</code>, the - * cache will use striped locks protecting a fixed array of buffers. - * This property is not understood by all implementations. - * - * @see Options#THREAD_LOCAL_BUFFERS - */ - public final boolean threadLocalBuffers; - - /** - * The initial capacity for the backing {@link ILRUCache} hash map for - * each {@link IRawStore}. - * - * @see Options#INITIAL_CAPACITY - */ - public final int initialCacheCapacity; - - /** - * The limiting cache capacity across all cache instances. - * - * @see Options#LIMITING_CAPACITY - */ - public final int limitingCacheCapacity; - - /** - * The percentage of the JVM heap to use for bigdata buffers. - * - * @see Options#PERCENT_HEAP - */ - public final float percentHeap; - - /** - * The maximum heap size in bytes (optional). - * - * @see Options#MAX_HEAP - */ - public final long maxHeap; - - /** - * The maximum bytesInMemory to retain across the caches. This is - * computed based on {@link #percentHeap} and {@link #maxHeap} and - * represents the actual limit which will be imposed on the shared LRU. - * - * @see Options#MAX_HEAP - * @see Options#PERCENT_HEAP - */ - public final long maximumBytesInMemory; - - /** - * @see Options#PERCENT_CLEARED - */ - public final double percentCleared; - - /** - * {@link #percentCleared} TIMES {@link #maximumBytesInMemory}. - */ - public final long minCleared; - - /** - * The minimum #of caches to keep open for an {@link IGlobalLRU} based - * on a weak value hash map. - * - * @see Options#MIN_CACHE_SET_SIZE - */ - public final int minCacheSetSize; - - /** - * @see WeakReferenceGlobalLRU - */ - public final int queueCapacity; - - /** - * @see WeakReferenceGlobalLRU - */ - public final int nscan; - - /** - * The capacity of the thread-local buffer used to amortize the cost of - * updating the access policy. This option is only recognized by some - * {@link IGlobalLRU} implementations. - * - * @see BCHMGlobalLRU2 - * - * @see Options#THREAD_LOCAL_BUFFER_CAPACITY - */ - public final int threadLocalBufferCapacity; - - /** - * The access policy algorithm (LRU, LIRS, etc). - * - * @see Options#ACCESS_POLICY - */ - public final AccessPolicyEnum accessPolicy; - - /** - * Parses the {@link Options} found in the caller's {@link Properties} - * to populate the fields of this {@link CacheSettings} object. - * - * @param properties - * The properties. - * @throws ClassNotFoundException - */ - public CacheSettings(final Properties properties) - throws ClassNotFoundException { - - if (properties == null) - throw new IllegalArgumentException(); - - enabled = Boolean.valueOf(properties.getProperty( - Options.ENABLED, Options.DEFAULT_ENABLED)); - - indexSegmentBuildPopulatesCache = Boolean.valueOf(properties.getProperty( - Options.INDEX_SEGMENT_BUILD_POPULATES_CACHE, - Options.DEFAULT_INDEX_SEGMENT_BUILD_POPULATES_CACHE)); - - cls = (Class<? extends IGlobalLRU>) LRUNexus.class - .forName(properties.getProperty(Options.CLASS, - Options.DEFAULT_CLASS)); - - final boolean validClass = IGlobalLRU.class.isAssignableFrom(cls); - - if (!validClass) { - - throw new RuntimeException("Class does not implement " - + IGlobalLRU.class.getName() + " : class=" + cls); - - } - - loadFactor = Float.valueOf(properties.getProperty( - Options.LOAD_FACTOR, Options.DEFAULT_LOAD_FACTOR)); - - concurrencyLevel = Integer.valueOf(properties.getProperty( - Options.CONCURRENCY_LEVEL, Options.DEFAULT_CONCURRENCY_LEVEL)); - - threadLocalBuffers = Boolean.valueOf(properties.getProperty( - Options.THREAD_LOCAL_BUFFERS, - Options.DEFAULT_THREAD_LOCAL_BUFFERS)); - - initialCacheCapacity = Integer.valueOf(System - .getProperty(Options.INITIAL_CAPACITY, - Options.DEFAULT_INITIAL_CAPACITY)); - - limitingCacheCapacity = Integer.valueOf(System - .getProperty(Options.LIMITING_CAPACITY, - Options.DEFAULT_LIMITING_CAPACITY)); - - threadLocalBufferCapacity = Integer.valueOf(properties.getProperty( - Options.THREAD_LOCAL_BUFFER_CAPACITY, - Options.DEFAULT_THREAD_LOCAL_BUFFER_CAPACITY)); - - accessPolicy = AccessPolicyEnum.valueOf(properties.getProperty( - Options.ACCESS_POLICY, Options.DEFAULT_ACCESS_POLICY)); - - percentHeap = Float.valueOf(properties.getProperty( - Options.PERCENT_HEAP, Options.DEFAULT_PERCENT_HEAP)); - - if (percentHeap < 0f || percentHeap > 1f) { - - throw new IllegalArgumentException(Options.PERCENT_HEAP - + " : must be in [0:1]."); - - } - - maxHeap = BytesUtil.getByteCount(properties.getProperty( - Options.MAX_HEAP, Options.DEFAULT_MAX_HEAP)); - - if (maxHeap < 0) - throw new IllegalArgumentException(Options.MAX_HEAP - + "=" - + properties.getProperty(Options.MAX_HEAP, - Options.DEFAULT_MAX_HEAP)); - - if (maxHeap == 0 && percentHeap != 0f) { - // compute based on the percentage of the heap. - maximumBytesInMemory = (long) (Runtime.getRuntime() - .maxMemory() * percentHeap); - } else if (maxHeap != 0) { - // directly given. - maximumBytesInMemory = maxHeap; - } else { - // disabled. - maximumBytesInMemory = 0L; - } - - percentCleared = Double.valueOf(properties.getProperty( - Options.PERCENT_CLEARED, Options.DEFAULT_PERCENT_CLEARED)); - - if (percentCleared < 0f || percentCleared > 1f) { - - throw new IllegalArgumentException(Options.PERCENT_CLEARED - + " : must be in [0:1]."); - - } - - minCleared = (long) (percentCleared * maximumBytesInMemory); - - minCacheSetSize = Integer.valueOf(properties.getProperty( - Options.MIN_CACHE_SET_SIZE, - Options.DEFAULT_MIN_CACHE_SET_SIZE)); - - /* - * Note: Values below this point are specific to the - * WeakReferenceGlobalLRU. - */ - - /* - * Estimate of the average record size. - * - * Note: 1024 is not a bad value for a WORM journal, but 4096 or - * 8192 are better values for the RW store and the index segment - * files. - */ - // The average record size. - final int baseAverageRecordSize = 1024; - - final int averageRecordSize = (int) (baseAverageRecordSize * (Integer - .valueOf(IndexMetadata.Options.DEFAULT_BTREE_BRANCHING_FACTOR) / 32.)); - - /* - * The target capacity for that expected record size. - * - * Note: This parameter can get you into trouble with too much GC if - * too much gets buffered on the queue (this is the reasons this LRU - * implementation is not recommended!) - * - * 4x may be a bit aggressive. Try 3x. - * - * TestTripleStoreLoadRateLocal: 4x yields 38s GC time with 1G heap. - * - * TestTripleStoreLoadRateLocal: 3x yields 36s GC time with 1G heap. - */ - final long maximumQueueCapacityEstimate = maximumBytesInMemory - / averageRecordSize * 2; - - if (BigdataStatics.debug) - System.err.println(// - "averageRecordSize=" - + averageRecordSize// - + ", maximumQueueCapacityEstimate=" - + maximumQueueCapacityEstimate// - ); - - if (true) { - - queueCapacity = (int) Math.min(Integer.MAX_VALUE, - maximumQueueCapacityEstimate); - - } else if (maximumBytesInMemory < Bytes.gigabyte * 2) { - - // capacity is no more than X - queueCapacity = (int) Math.min( - maximumQueueCapacityEstimate, 200000/* - * 200k - */); - - } else { - - // capacity is no more than Y - queueCapacity = (int) Math.min( - maximumQueueCapacityEstimate, 1000000/* - * 1M - */); - - } - - nscan = 20; -// Integer.valueOf(properties.getProperty(Options.NSCAN, -// Options.DEFAULT_NSCAN)); - - } - - /** - * Create a new {@link IGlobalLRU} instance from the - * {@link CacheSettings}. The {@link IGlobalLRU} MUST define a public - * constructor with the following method signature. - * - * <pre> - * public FooGlobalLRU(CacheSettings) - * </pre> - * - * @return The new instance -or- <code>null</code> if the cache is - * disabled. - * - * @throws NoSuchMethodException - * @throws SecurityException - * @throws InvocationTargetException - * @throws IllegalAccessException - * @throws InstantiationException - * @throws IllegalArgumentException - * @throws UnsupportedOperationException - * if something is not supported.... - * - * @todo Instead of returning <code>null</code> if the cache is not - * enabled or if something goes wrong we could return a - * NOPGlobalLRU. That could simplify conditional logic. The - * implementation would have to support per-store caches but would - * not retain any records in those caches. - */ - public IGlobalLRU<Long, Object> newInstance() throws SecurityException, - NoSuchMethodException, IllegalArgumentException, - InstantiationException, IllegalAccessException, - InvocationTargetException { - - if (enabled) { - - if (maximumBytesInMemory > 0) { - - final Constructor<?> ctor = cls - .getConstructor(new Class[] { CacheSettings.class }); - - return (IGlobalLRU<Long, Object>) ctor - .newInstance(new Object[] { this }); - - } - - } - - // Not enabled. - return null; - - } - - public String toString() { - - return super.toString() - + "{"// - + "maxPercent=" + percentHeap// - + ", maxHeap=" + maxHeap// - + ", maximumBytesInMemory=" + maximumBytesInMemory// - + ", percentCleared=" + percentCleared// - + ", minCleared=" + minCleared// - + ", maxMemory=" + Runtime.getRuntime().maxMemory()// - + ", loadFactor=" + loadFactor// - + ", concurrencyLevel=" + concurrencyLevel// - + ", threadLocalBuffers=" + threadLocalBuffers// - + ", threadLocalBufferCapacity=" + threadLocalBufferCapacity// - + ", initialCacheCapacity=" + initialCacheCapacity// - + ", limitingCacheCapacity=" + limitingCacheCapacity// - + ", minCacheSetSize=" + minCacheSetSize// - + ", queueCapacity=" + queueCapacity// - + ", nscan=" + nscan// - + ", cls=" + cls.getName()// - + ", indexSegmentBuildPopulatesCache=" + indexSegmentBuildPopulatesCache + // - "}"; - - } - - } - - /** - * The configuration in use. - */ - private static final CacheSettings settings; - - static { - - IGlobalLRU<Long, Object> tmp = null; - CacheSettings s = null; - - try { - - // parse the options. - s = new CacheSettings(System.getProperties()); - - if (BigdataStatics.debug || log.isInfoEnabled()) { - - final String msg = s.enabled ? s.toString() - : "LRUNexus is disabled"; - - if (BigdataStatics.debug) - System.err.println(msg); - - if (log.isInfoEnabled()) - log.info(msg); - - } - - // create the cache object. - tmp = s.newInstance(); - - } catch (Throwable t) { - - log.error("LRUNexus disabled", t); - - } finally { - - // Note: MAY be null. - INSTANCE = tmp; - - // Note: MAY be null. - settings = s; - - } - - } - - /** - * Return <code>true</code> if the {@link IndexSegmentBuilder} will populate - * the {@link IGlobalLRU} with records for the new {@link IndexSegment} - * during the build. - * - * @see Options#INDEX_SEGMENT_BUILD_POPULATES_CACHE - */ - public static final boolean getIndexSegmentBuildPopulatesCache() { - - return settings != null && settings.indexSegmentBuildPopulatesCache; - - } - - /** - * Factory returns the {@link ILRUCache} for the store iff the - * {@link LRUNexus} is enabled. - * - * @param store - * The store. - * - * @return The cache for that store if the {@link LRUNexus} is enabled and - * otherwise <code>null</code>. - * - * @throws IllegalArgumentException - * if the store is <code>null</code>. - */ - public static ILRUCache<Long, Object> getCache(final IRawStore store) { - - if (store == null) - throw new IllegalArgumentException(); - - if (INSTANCE == null) - return null; - - final IAddressManager am; - - if (store instanceof AbstractJournal) { - - /* - * This avoids hard reference to the journal (it winds up using a - * clone of the address manager instead). - */ - - am = ((IBufferStrategy) ((AbstractJournal) store) - .getBufferStrategy()).getAddressManager(); - - } else if (store instanceof TemporaryRawStore) { - - /* - * This avoids using a hard reference to the temporary store (it - * basically clones the address manager instead). - */ - - am = new WormAddressManager(((TemporaryRawStore) store) - .getOffsetBits()); - - } else if (store instanceof AbstractRawStore) { - - /* - * Note: this covers the IndexSegmentStore. - */ - am = ((AbstractRawStore) store).getAddressManager(); - - } else { - - // @todo which cases come though here? SimpleMemoryStore, - // SimpleFileStore, - am = null; - - } - - if (am instanceof IRawStore) { - - /* - * This would cause the IRawStore to be retained by a hard - * reference! - */ - - throw new AssertionError(am.getClass().getName() + " implements " - + IRawStore.class.getName()); - - } - - return INSTANCE.getCache(store.getUUID(), am); - - } - - /** - * Command line utility may be used to confirm the environment settings. - * - * @param args - * Ignored. All parameters are specified either in the - * environment or using JVM - * <code>-Dcom.bigdata.LRUNexus.foo=bar</code> arguments on the - * command line. - * - * @throws ClassNotFoundException - */ - public static void main(String[] args) throws ClassNotFoundException { - - System.out.println(new CacheSettings(System.getProperties()).toString()); - - } - -} Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/AbstractBTree.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/AbstractBTree.java 2010-09-20 21:37:58 UTC (rev 3600) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/AbstractBTree.java 2010-09-20 23:46:12 UTC (rev 3601) @@ -47,7 +47,6 @@ import com.bigdata.Banner; import com.bigdata.BigdataStatics; -import com.bigdata.LRUNexus; import com.bigdata.btree.AbstractBTreeTupleCursor.MutableBTreeTupleCursor; import com.bigdata.btree.AbstractBTreeTupleCursor.ReadOnlyBTreeTupleCursor; import com.bigdata.btree.IndexMetadata.Options; @@ -244,12 +243,6 @@ final protected IRawStore store; /** - * Optional cache for {@link INodeData} and {@link ILeafData} instances and - * always <code>null</code> if the B+Tree is transient. - */ - protected final ILRUCache<Long, Object> storeCache; - - /** * The branching factor for the btree. */ final protected int branchingFactor; @@ -645,12 +638,6 @@ * held). This means that lookup in a map is not required for top-down * navigation. * <p> - * The {@link LRUNexus} provides an {@link INodeData} / {@link ILeafData} - * data record cache based on a hash map with lookup by the address of the - * node or leaf. This is tested when the child {@link WeakReference} was - * never set or has been cleared. This cache is also used by the - * {@link IndexSegment} for the linked-leaf traversal pattern, which does - * not use top-down navigation. * * @todo consider a policy that dynamically adjusts the queue capacities * based on the height of the btree in order to maintain a cache that @@ -952,11 +939,7 @@ * the child nodes and the parents are connected using hard links * rather than weak references. */ - - this.storeCache = null; -// this.globalLRU = null; - // this.readRetentionQueue = null; } else { @@ -972,16 +955,6 @@ * support concurrent read operations. The INodeData or ILeafData * will be wrapped as a Node or Leaf by the owning B+Tree instance. */ - - ... [truncated message content] |
From: <res...@us...> - 2010-09-30 13:43:39
|
Revision: 3691 http://bigdata.svn.sourceforge.net/bigdata/?rev=3691&view=rev Author: resendes Date: 2010-09-30 13:43:30 +0000 (Thu, 30 Sep 2010) Log Message: ----------- Merge (back) from bbb_cleanup branch -- fixes for com.bigdata.util package Modified Paths: -------------- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/BootStateUtil.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/CSVReader.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/EntryUtil.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/Format.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/HTMLUtility.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/InnerCause.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/MillisecondTimestampFactory.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/NT.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/NV.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/ReverseLongComparator.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/util/TestAll.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/util/TestCSVReader.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/util/TestInnerCause.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/util/TestMillisecondTimestampFactory.java Added Paths: ----------- branches/maven_scaleout/bigdata-core/build/ branches/maven_scaleout/bigdata-core/build/classes/ branches/maven_scaleout/bigdata-core/src/META-INF/ branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/util/CSVReaderBuilder.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/util/TestBootStateUtil.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/util/TestEntryUtil.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/util/TestFormat.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/util/TestHTMLUtility.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/util/TestNT.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/util/TestNV.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/util/TestReverseLongComparator.java Removed Paths: ------------- branches/maven_scaleout/bigdata-core/build/classes/ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/ByteBufferBitVector.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/ChecksumError.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/ClassLoaderUtil.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/HybridTimestampFactory.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/NanosecondTimestampFactory.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/util/TestByteBufferBitVector.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/util/TestHybridTimestampFactory.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/util/TestNanosecondTimestampFactory.java Property Changed: ---------------- branches/maven_scaleout/bigdata-core/ branches/maven_scaleout/bigdata-core/bigdata-perf/ branches/maven_scaleout/bigdata-core/bigdata-perf/lubm/src/resources/ branches/maven_scaleout/bigdata-core/dsi-utils/LEGAL/ branches/maven_scaleout/bigdata-core/dsi-utils/lib/ branches/maven_scaleout/bigdata-core/dsi-utils/src/ branches/maven_scaleout/bigdata-core/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom/ branches/maven_scaleout/bigdata-core/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom/ branches/maven_scaleout/bigdata-core/osgi/ branches/maven_scaleout/bigdata-core/src/main/deploy/bin/ branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/jini/ branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster.config branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/jini/boot/ branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/jini/shardlocator.config branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/logging/ branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/logging/shardlocator-logging.properties branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/logging/transaction-logging.properties branches/maven_scaleout/bigdata-core/src/main/java/ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/attr/ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/disco/ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/rdf/sail/bench/ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/rdf/util/ branches/maven_scaleout/bigdata-core/src/samples-sail/com/bigdata/samples/fastload.properties branches/maven_scaleout/bigdata-core/src/test/deploy/testing/conf/bigdataStandaloneTesting.config branches/maven_scaleout/bigdata-core/src/test/deploy/testing/data/com/bigdata/jini/start/config/testfed.config branches/maven_scaleout/bigdata-core/src/test/deploy/testing/data/com/bigdata/jini/start/testfed.config branches/maven_scaleout/bigdata-core/src/test/deploy/testing/data/com/bigdata/service/jini/master/TestMappedRDFDataLoadMaster.config branches/maven_scaleout/bigdata-core/src/test/java/ Property changes on: branches/maven_scaleout/bigdata-core ___________________________________________________________________ Modified: svn:mergeinfo - /trunk:3499 + /branches/bbb_cleanup/bigdata-core:3588-3690 /trunk:3499 Property changes on: branches/maven_scaleout/bigdata-core/bigdata-perf ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/bigdata-perf:3379-3541 + /branches/bbb_cleanup/bigdata-core/bigdata-perf:3588-3690 /trunk/bigdata-perf:3379-3541 Property changes on: branches/maven_scaleout/bigdata-core/bigdata-perf/lubm/src/resources ___________________________________________________________________ Modified: svn:mergeinfo - /branches/dev-btm/bigdata-perf/lubm/src/resources:2574-3440 /trunk/bigdata-perf/lubm/src/resources:3379-3541 + /branches/bbb_cleanup/bigdata-core/bigdata-perf/lubm/src/resources:3588-3690 /branches/dev-btm/bigdata-perf/lubm/src/resources:2574-3440 /trunk/bigdata-perf/lubm/src/resources:3379-3541 Property changes on: branches/maven_scaleout/bigdata-core/dsi-utils/LEGAL ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/dsi-utils/LEGAL:3379-3430,3499 + /branches/bbb_cleanup/bigdata-core/dsi-utils/LEGAL:3588-3690 /trunk/dsi-utils/LEGAL:3379-3430,3499 Property changes on: branches/maven_scaleout/bigdata-core/dsi-utils/lib ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/dsi-utils/lib:3379-3430,3499 + /branches/bbb_cleanup/bigdata-core/dsi-utils/lib:3588-3690 /trunk/dsi-utils/lib:3379-3430,3499 Property changes on: branches/maven_scaleout/bigdata-core/dsi-utils/src ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/dsi-utils/src:3379-3430,3499 + /branches/bbb_cleanup/bigdata-core/dsi-utils/src:3588-3690 /trunk/dsi-utils/src:3379-3430,3499 Property changes on: branches/maven_scaleout/bigdata-core/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom:3379-3430,3499 + /branches/bbb_cleanup/bigdata-core/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom:3588-3690 /trunk/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom:3379-3430,3499 Property changes on: branches/maven_scaleout/bigdata-core/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom:3379-3430,3499 + /branches/bbb_cleanup/bigdata-core/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom:3588-3690 /trunk/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom:3379-3430,3499 Property changes on: branches/maven_scaleout/bigdata-core/osgi ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/osgi:3379-3430,3499 + /branches/bbb_cleanup/bigdata-core/osgi:3588-3690 /trunk/osgi:3379-3430,3499 Property changes on: branches/maven_scaleout/bigdata-core/src/main/deploy/bin ___________________________________________________________________ Modified: svn:mergeinfo - /branches/BTREE_BUFFER_BRANCH/bigdata-core/src/main/deploy/bin:2004-2045 /branches/DEV_BRANCH_27_OCT_2009/bigdata-core/src/main/deploy/bin:2270-2546,2548-2782 /branches/LEXICON_REFACTOR_BRANCH/bigdata-core/src/main/deploy/bin:2633-3304 /branches/bugfix-btm/bigdata-core/src/main/deploy/bin:2594-3237 /branches/dev-btm/bigdata-core/src/main/deploy/bin:2574-3440,3443,3463,3469-3470 /branches/dev-btm/src/resources/bin:3463 /branches/fko/bigdata-core/src/main/deploy/bin:3150-3194 /trunk/bigdata-core/src/main/deploy/bin:3379-3430,3432-3460,3476-3499,3503,3507,3516-3528,3542 + /branches/BTREE_BUFFER_BRANCH/bigdata-core/src/main/deploy/bin:2004-2045 /branches/DEV_BRANCH_27_OCT_2009/bigdata-core/src/main/deploy/bin:2270-2546,2548-2782 /branches/LEXICON_REFACTOR_BRANCH/bigdata-core/src/main/deploy/bin:2633-3304 /branches/bbb_cleanup/bigdata-core/src/main/deploy/bin:3588-3690 /branches/bugfix-btm/bigdata-core/src/main/deploy/bin:2594-3237 /branches/dev-btm/bigdata-core/src/main/deploy/bin:2574-3440,3443,3463,3469-3470 /branches/dev-btm/src/resources/bin:3463 /branches/fko/bigdata-core/src/main/deploy/bin:3150-3194 /trunk/bigdata-core/src/main/deploy/bin:3379-3430,3432-3460,3476-3499,3503,3507,3516-3528,3542 Property changes on: branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/jini ___________________________________________________________________ Modified: svn:mergeinfo - /trunk/src/main/deploy/var/config/jini:3499 /trunk/src/resources/config:3516-3528 + /branches/bbb_cleanup/bigdata-core/src/main/deploy/var/config/jini:3588-3690 /trunk/src/main/deploy/var/config/jini:3499 /trunk/src/resources/config:3516-3528 Property changes on: branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster.config ___________________________________________________________________ Modified: svn:mergeinfo - /branches/BTREE_BUFFER_BRANCH/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster.config:2004-2045 /branches/DEV_BRANCH_27_OCT_2009/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster.config:2270-2546,2548-2782 /branches/LEXICON_REFACTOR_BRANCH/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster.config:2633-3304 /branches/bugfix-btm/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster.config:2594-3237 /branches/dev-btm/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster.config:2574-3440,3443,3463,3469-3470 /branches/dev-btm/src/resources/config/bigdataCluster.config:3463 /branches/fko/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster.config:3150-3194 /trunk/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster.config:3379-3430,3432-3460,3476-3499,3503,3507,3516-3528,3542 /trunk/src/resources/config/bigdataCluster.config:3516-3528 + /branches/BTREE_BUFFER_BRANCH/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster.config:2004-2045 /branches/DEV_BRANCH_27_OCT_2009/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster.config:2270-2546,2548-2782 /branches/LEXICON_REFACTOR_BRANCH/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster.config:2633-3304 /branches/bbb_cleanup/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster.config:3588-3690 /branches/bugfix-btm/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster.config:2594-3237 /branches/dev-btm/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster.config:2574-3440,3443,3463,3469-3470 /branches/dev-btm/src/resources/config/bigdataCluster.config:3463 /branches/fko/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster.config:3150-3194 /trunk/bigdata-core/src/main/deploy/var/config/jini/bigdataCluster.config:3379-3430,3432-3460,3476-3499,3503,3507,3516-3528,3542 /trunk/src/resources/config/bigdataCluster.config:3516-3528 Property changes on: branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/jini/boot ___________________________________________________________________ Modified: svn:mergeinfo - /branches/BTREE_BUFFER_BRANCH/bigdata-core/src/main/deploy/var/config/jini/boot:2004-2045 /branches/DEV_BRANCH_27_OCT_2009/bigdata-core/src/main/deploy/var/config/jini/boot:2270-2546,2548-2782 /branches/LEXICON_REFACTOR_BRANCH/bigdata-core/src/main/deploy/var/config/jini/boot:2633-3304 /branches/bugfix-btm/bigdata-core/src/main/deploy/var/config/jini/boot:2594-3237 /branches/dev-btm/bigdata-core/src/main/deploy/var/config/jini/boot:2574-3440,3443,3463,3469-3470 /branches/dev-btm/bigdata-jini/src/java/com/bigdata/boot/config:3469-3470 /branches/fko/bigdata-core/src/main/deploy/var/config/jini/boot:3150-3194 /branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/jini/boot:3379-3438 /trunk/bigdata-core/src/main/deploy/var/config/jini/boot:3379-3430,3432-3460,3476-3499,3503,3507,3516-3528,3542 + /branches/BTREE_BUFFER_BRANCH/bigdata-core/src/main/deploy/var/config/jini/boot:2004-2045 /branches/DEV_BRANCH_27_OCT_2009/bigdata-core/src/main/deploy/var/config/jini/boot:2270-2546,2548-2782 /branches/LEXICON_REFACTOR_BRANCH/bigdata-core/src/main/deploy/var/config/jini/boot:2633-3304 /branches/bbb_cleanup/bigdata-core/src/main/deploy/var/config/jini/boot:3588-3690 /branches/bugfix-btm/bigdata-core/src/main/deploy/var/config/jini/boot:2594-3237 /branches/dev-btm/bigdata-core/src/main/deploy/var/config/jini/boot:2574-3440,3443,3463,3469-3470 /branches/dev-btm/bigdata-jini/src/java/com/bigdata/boot/config:3469-3470 /branches/fko/bigdata-core/src/main/deploy/var/config/jini/boot:3150-3194 /branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/jini/boot:3379-3438 /trunk/bigdata-core/src/main/deploy/var/config/jini/boot:3379-3430,3432-3460,3476-3499,3503,3507,3516-3528,3542 Property changes on: branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/jini/shardlocator.config ___________________________________________________________________ Modified: svn:mergeinfo - /branches/BTREE_BUFFER_BRANCH/bigdata-core/src/main/deploy/var/config/jini/shardlocator.config:2004-2045 /branches/DEV_BRANCH_27_OCT_2009/bigdata-core/src/main/deploy/var/config/jini/shardlocator.config:2270-2546,2548-2782 /branches/LEXICON_REFACTOR_BRANCH/bigdata-core/src/main/deploy/var/config/jini/shardlocator.config:2633-3304 /branches/bugfix-btm/bigdata-core/src/main/deploy/var/config/jini/shardlocator.config:2594-3237 /branches/dev-btm/bigdata-core/src/main/deploy/var/config/jini/shardlocator.config:2574-3440,3443,3463,3469-3470 /branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/shardlocator.config:3463 /branches/fko/bigdata-core/src/main/deploy/var/config/jini/shardlocator.config:3150-3194 /trunk/bigdata-core/src/main/deploy/var/config/jini/shardlocator.config:3379-3430,3432-3460,3476-3499,3503,3507,3516-3528,3542 + /branches/BTREE_BUFFER_BRANCH/bigdata-core/src/main/deploy/var/config/jini/shardlocator.config:2004-2045 /branches/DEV_BRANCH_27_OCT_2009/bigdata-core/src/main/deploy/var/config/jini/shardlocator.config:2270-2546,2548-2782 /branches/LEXICON_REFACTOR_BRANCH/bigdata-core/src/main/deploy/var/config/jini/shardlocator.config:2633-3304 /branches/bbb_cleanup/bigdata-core/src/main/deploy/var/config/jini/shardlocator.config:3588-3690 /branches/bugfix-btm/bigdata-core/src/main/deploy/var/config/jini/shardlocator.config:2594-3237 /branches/dev-btm/bigdata-core/src/main/deploy/var/config/jini/shardlocator.config:2574-3440,3443,3463,3469-3470 /branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/shardlocator.config:3463 /branches/fko/bigdata-core/src/main/deploy/var/config/jini/shardlocator.config:3150-3194 /trunk/bigdata-core/src/main/deploy/var/config/jini/shardlocator.config:3379-3430,3432-3460,3476-3499,3503,3507,3516-3528,3542 Property changes on: branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/logging ___________________________________________________________________ Modified: svn:mergeinfo - /branches/BTREE_BUFFER_BRANCH/bigdata-core/src/main/deploy/var/config/logging:2004-2045 /branches/DEV_BRANCH_27_OCT_2009/bigdata-core/src/main/deploy/var/config/logging:2270-2546,2548-2782 /branches/LEXICON_REFACTOR_BRANCH/bigdata-core/src/main/deploy/var/config/logging:2633-3304 /branches/bugfix-btm/bigdata-core/src/main/deploy/var/config/logging:2594-3237 /branches/dev-btm/bigdata/src/resources/logging:3463 /branches/dev-btm/bigdata-core/src/main/deploy/var/config/logging:2574-3440,3443,3463,3469-3470 /branches/fko/bigdata-core/src/main/deploy/var/config/logging:3150-3194 /trunk/bigdata-core/src/main/deploy/var/config/logging:3379-3430,3432-3460,3476-3499,3503,3507,3516-3528,3542 + /branches/BTREE_BUFFER_BRANCH/bigdata-core/src/main/deploy/var/config/logging:2004-2045 /branches/DEV_BRANCH_27_OCT_2009/bigdata-core/src/main/deploy/var/config/logging:2270-2546,2548-2782 /branches/LEXICON_REFACTOR_BRANCH/bigdata-core/src/main/deploy/var/config/logging:2633-3304 /branches/bbb_cleanup/bigdata-core/src/main/deploy/var/config/logging:3588-3690 /branches/bugfix-btm/bigdata-core/src/main/deploy/var/config/logging:2594-3237 /branches/dev-btm/bigdata/src/resources/logging:3463 /branches/dev-btm/bigdata-core/src/main/deploy/var/config/logging:2574-3440,3443,3463,3469-3470 /branches/fko/bigdata-core/src/main/deploy/var/config/logging:3150-3194 /trunk/bigdata-core/src/main/deploy/var/config/logging:3379-3430,3432-3460,3476-3499,3503,3507,3516-3528,3542 Property changes on: branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/logging/shardlocator-logging.properties ___________________________________________________________________ Modified: svn:mergeinfo - /branches/BTREE_BUFFER_BRANCH/bigdata-core/src/main/deploy/var/config/logging/shardlocator-logging.properties:2004-2045 /branches/DEV_BRANCH_27_OCT_2009/bigdata-core/src/main/deploy/var/config/logging/shardlocator-logging.properties:2270-2546,2548-2782 /branches/LEXICON_REFACTOR_BRANCH/bigdata-core/src/main/deploy/var/config/logging/shardlocator-logging.properties:2633-3304 /branches/bugfix-btm/bigdata-core/src/main/deploy/var/config/logging/shardlocator-logging.properties:2594-3237 /branches/dev-btm/bigdata/src/resources/logging/shardlocator-logging.properties:3463 /branches/dev-btm/bigdata-core/src/main/deploy/var/config/logging/shardlocator-logging.properties:2574-3440,3443,3463,3469-3470 /branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/logging.properties:3463 /branches/fko/bigdata-core/src/main/deploy/var/config/logging/shardlocator-logging.properties:3150-3194 /trunk/bigdata-core/src/main/deploy/var/config/logging/shardlocator-logging.properties:3379-3430,3432-3460,3476-3499,3503,3507,3516-3528,3542 + /branches/BTREE_BUFFER_BRANCH/bigdata-core/src/main/deploy/var/config/logging/shardlocator-logging.properties:2004-2045 /branches/DEV_BRANCH_27_OCT_2009/bigdata-core/src/main/deploy/var/config/logging/shardlocator-logging.properties:2270-2546,2548-2782 /branches/LEXICON_REFACTOR_BRANCH/bigdata-core/src/main/deploy/var/config/logging/shardlocator-logging.properties:2633-3304 /branches/bbb_cleanup/bigdata-core/src/main/deploy/var/config/logging/shardlocator-logging.properties:3588-3690 /branches/bugfix-btm/bigdata-core/src/main/deploy/var/config/logging/shardlocator-logging.properties:2594-3237 /branches/dev-btm/bigdata/src/resources/logging/shardlocator-logging.properties:3463 /branches/dev-btm/bigdata-core/src/main/deploy/var/config/logging/shardlocator-logging.properties:2574-3440,3443,3463,3469-3470 /branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/logging.properties:3463 /branches/fko/bigdata-core/src/main/deploy/var/config/logging/shardlocator-logging.properties:3150-3194 /trunk/bigdata-core/src/main/deploy/var/config/logging/shardlocator-logging.properties:3379-3430,3432-3460,3476-3499,3503,3507,3516-3528,3542 Property changes on: branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/logging/transaction-logging.properties ___________________________________________________________________ Modified: svn:mergeinfo - /branches/BTREE_BUFFER_BRANCH/bigdata-core/src/main/deploy/var/config/logging/transaction-logging.properties:2004-2045 /branches/DEV_BRANCH_27_OCT_2009/bigdata-core/src/main/deploy/var/config/logging/transaction-logging.properties:2270-2546,2548-2782 /branches/LEXICON_REFACTOR_BRANCH/bigdata-core/src/main/deploy/var/config/logging/transaction-logging.properties:2633-3304 /branches/bugfix-btm/bigdata-core/src/main/deploy/var/config/logging/transaction-logging.properties:2594-3237 /branches/dev-btm/bigdata/src/resources/logging/transaction-logging.properties:3463 /branches/dev-btm/bigdata-core/src/main/deploy/var/config/logging/transaction-logging.properties:2574-3440,3443,3463,3469-3470 /branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/config/logging.properties:3463 /branches/fko/bigdata-core/src/main/deploy/var/config/logging/transaction-logging.properties:3150-3194 /trunk/bigdata-core/src/main/deploy/var/config/logging/transaction-logging.properties:3379-3430,3432-3460,3476-3499,3503,3507,3516-3528,3542 + /branches/BTREE_BUFFER_BRANCH/bigdata-core/src/main/deploy/var/config/logging/transaction-logging.properties:2004-2045 /branches/DEV_BRANCH_27_OCT_2009/bigdata-core/src/main/deploy/var/config/logging/transaction-logging.properties:2270-2546,2548-2782 /branches/LEXICON_REFACTOR_BRANCH/bigdata-core/src/main/deploy/var/config/logging/transaction-logging.properties:2633-3304 /branches/bbb_cleanup/bigdata-core/src/main/deploy/var/config/logging/transaction-logging.properties:3588-3690 /branches/bugfix-btm/bigdata-core/src/main/deploy/var/config/logging/transaction-logging.properties:2594-3237 /branches/dev-btm/bigdata/src/resources/logging/transaction-logging.properties:3463 /branches/dev-btm/bigdata-core/src/main/deploy/var/config/logging/transaction-logging.properties:2574-3440,3443,3463,3469-3470 /branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/config/logging.properties:3463 /branches/fko/bigdata-core/src/main/deploy/var/config/logging/transaction-logging.properties:3150-3194 /trunk/bigdata-core/src/main/deploy/var/config/logging/transaction-logging.properties:3379-3430,3432-3460,3476-3499,3503,3507,3516-3528,3542 Property changes on: branches/maven_scaleout/bigdata-core/src/main/java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/BTREE_BUFFER_BRANCH/bigdata-core/src/main/java:2004-2045 /branches/DEV_BRANCH_27_OCT_2009/bigdata-core/src/main/java:2270-2546,2548-2782 /branches/LEXICON_REFACTOR_BRANCH/bigdata-core/src/main/java:2633-3304 /branches/bugfix-btm/bigdata-core/src/main/java:2594-3237 /branches/dev-btm/bigdata/src/java:3463 /branches/dev-btm/bigdata-core/src/main/java:2574-3440,3443,3463,3469-3470 /branches/dev-btm/bigdata-jini/src/java:3463,3469-3470 /branches/dev-btm/bigdata-rdf/src/java:3463 /branches/dev-btm/bigdata-sails/src/java:3463 /branches/fko/bigdata-core/src/main/java:3150-3194 /trunk/bigdata/src/java:3507 /trunk/bigdata-core/src/main/java:3379-3430,3432-3460,3476-3499,3503,3507,3516-3528,3542 /trunk/bigdata-rdf/src/java:3542 + /branches/BTREE_BUFFER_BRANCH/bigdata-core/src/main/java:2004-2045 /branches/DEV_BRANCH_27_OCT_2009/bigdata-core/src/main/java:2270-2546,2548-2782 /branches/LEXICON_REFACTOR_BRANCH/bigdata-core/src/main/java:2633-3304 /branches/bbb_cleanup/bigdata-core/src/main/java:3588-3690 /branches/bugfix-btm/bigdata-core/src/main/java:2594-3237 /branches/dev-btm/bigdata/src/java:3463 /branches/dev-btm/bigdata-core/src/main/java:2574-3440,3443,3463,3469-3470 /branches/dev-btm/bigdata-jini/src/java:3463,3469-3470 /branches/dev-btm/bigdata-rdf/src/java:3463 /branches/dev-btm/bigdata-sails/src/java:3463 /branches/fko/bigdata-core/src/main/java:3150-3194 /trunk/bigdata/src/java:3507 /trunk/bigdata-core/src/main/java:3379-3430,3432-3460,3476-3499,3503,3507,3516-3528,3542 /trunk/bigdata-rdf/src/java:3542 Property changes on: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/attr ___________________________________________________________________ Modified: svn:mergeinfo - /branches/dev-btm/bigdata-jini/src/java/com/bigdata/attr:3463,3469-3470 /trunk/bigdata-jini/src/java/com/bigdata/attr:3379-3430 + /branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/attr:3588-3690 /branches/dev-btm/bigdata-jini/src/java/com/bigdata/attr:3463,3469-3470 /trunk/bigdata-jini/src/java/com/bigdata/attr:3379-3430 Property changes on: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/disco ___________________________________________________________________ Modified: svn:mergeinfo - /branches/dev-btm/bigdata-jini/src/java/com/bigdata/disco:3463,3469-3470 /trunk/bigdata-jini/src/java/com/bigdata/disco:3379-3430 + /branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/disco:3588-3690 /branches/dev-btm/bigdata-jini/src/java/com/bigdata/disco:3463,3469-3470 /trunk/bigdata-jini/src/java/com/bigdata/disco:3379-3430 Property changes on: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/rdf/sail/bench ___________________________________________________________________ Modified: svn:mergeinfo - /branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/bench:3463 /trunk/bigdata-sails/src/java/com/bigdata/rdf/sail/bench:3379-3430 + /branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/rdf/sail/bench:3588-3690 /branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/bench:3463 /trunk/bigdata-sails/src/java/com/bigdata/rdf/sail/bench:3379-3430 Property changes on: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/rdf/util ___________________________________________________________________ Modified: svn:mergeinfo - /branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util:3463 /trunk/bigdata-rdf/src/java/com/bigdata/rdf/util:3379-3430,3542 + /branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/rdf/util:3588-3690 /branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util:3463 /trunk/bigdata-rdf/src/java/com/bigdata/rdf/util:3379-3430,3542 Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/BootStateUtil.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/BootStateUtil.java 2010-09-30 11:47:53 UTC (rev 3690) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/BootStateUtil.java 2010-09-30 13:43:30 UTC (rev 3691) @@ -45,7 +45,6 @@ import com.sun.jini.config.Config; import net.jini.config.Configuration; import net.jini.config.ConfigurationException; -import net.jini.config.NoSuchEntryException; import net.jini.core.lookup.ServiceID; /** @@ -80,7 +79,7 @@ private File persistenceDir = null; private UUID proxyId = null; private ServiceID serviceId = null; - private String stateKey = null; + private String stateKey = null; //TODO -- not used? public BootStateUtil(final Configuration config, final String componentName, @@ -128,22 +127,6 @@ recoverBootState(defaultServiceId); } - - public BootStateUtil(File persistenceDir, - Class entityImplType, - ServiceID defaultServiceId) - throws IOException, ClassNotFoundException - { - if(entityImplType == null) { - throw new NullPointerException("entityImplType null"); - } - this.entityImplType = entityImplType; - this.logger = Logger.getLogger(this.getClass()); - this.persistenceDir = persistenceDir; - - recoverBootState(defaultServiceId); - } - /** * Returns the entity's unique <i>proxy id</i> that is generated/recoverd * as part of the boot state maintained by this class. @@ -161,44 +144,6 @@ return serviceId; } - /** - * Returns the <code>String</code> representing the path of the - * directory in which the entity's <i>boot state</i> is located. - * If the value returned is <code>null</code>, then the entity - * was configured to run in <i>transient</i> mode. - */ - public String getPersistenceDirectory() { - return (persistenceDir != null) ? persistenceDir.toString() : null; - } - - /** - * Returns the <code>true</code> if the entity was configured to - * run in <i>persistent</i> mode; <code>false</code> otherwise. - */ - public boolean isPersistent() { - return (persistenceDir != null); - } - - /** - * If the entity is currently configured to run in <i>persistent</i> - * mode, returns the name-based key under which an entity's (non-boot) - * state was persisted during previous runs of the entity. If the - * entity is currently configured to run in <i>transient</i> mode, - * a non-<code>null</code>, randomly-generated key value is returned. - */ - public String getStateKey() { - return stateKey; - } - - /** - * Returns the <code>Class</code> type of the entity whose boot state - * is associated with the current instance of this utility. - */ - public Class getType() { - return entityImplType; - } - - /* Performs the actual retrieval of the entity's boot state. This * method is called only once, in this utility's constructor. * It recovers the entity's boot state from local persistence storage, @@ -338,6 +283,7 @@ /** * @see java.io.ObjectInputStream#resolveClass */ + @Override protected Class<?> resolveClass(ObjectStreamClass desc) throws IOException, ClassNotFoundException { @@ -354,7 +300,7 @@ * An interface is specified here to support evolution of new * versions of BootState. */ - interface BootState { + private interface BootState { Class getType(); UUID getProxyId(); String getKey(); Deleted: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/ByteBufferBitVector.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/ByteBufferBitVector.java 2010-09-30 11:47:53 UTC (rev 3690) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/ByteBufferBitVector.java 2010-09-30 13:43:30 UTC (rev 3691) @@ -1,167 +0,0 @@ -/* - -Copyright (C) SYSTAP, LLC 2006-2008. All rights reserved. - -Contact: - SYSTAP, LLC - 4501 Tower Road - Greensboro, NC 27410 - lic...@bi... - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ -/* - * Created on Aug 6, 2009 - */ - -package com.bigdata.util; - -import it.unimi.dsi.bits.AbstractBitVector; - -import java.nio.ByteBuffer; - -import cern.colt.bitvector.BitVector; - -/** - * Wraps a {@link ByteBuffer} as a read-only {@link BitVector}. - * - * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ - */ -public class ByteBufferBitVector extends AbstractBitVector { - - /** - * The {@link ByteBuffer} containing the backing data. - */ - final private ByteBuffer b; - - /** - * The #of bits in the vector. - */ - private final long len; - - /** - * The bit offset into the {@link ByteBuffer} of the first bit in the - * vector. - */ - private final long off; - - final public long length() { - - return len; - - } - - /** - * Ctor assumes that all bits in the buffer are used. - * - * @param b - * The buffer. - */ - public ByteBufferBitVector(final ByteBuffer b) { - - this(b, 0/* offset */, b == null ? 0 : b.capacity() * 8/* len */); - - } - - /** - * - * @param b - * The buffer. - * @param off - * The offset from the start of the buffer for the view. - * @param len - * The #of bits which will be included in the view. - */ - public ByteBufferBitVector(final ByteBuffer b, final long off, - final long len) { - - if (b == null) - throw new IllegalArgumentException(); - - if (len < 0) - throw new IllegalArgumentException(); - - if (len < 0) - throw new IllegalArgumentException(); - - if (off + len > b.capacity() * 8L) - throw new IllegalArgumentException(); - - this.b = b; - - this.len = len; - - this.off = off; - - } - - /** - * Return the index of the byte in which the bit with the given index is - * encoded. - * - * @param bitIndex - * The bit index. - * - * @return The byte index. - */ - final protected int byteIndexForBit(final long bitIndex) { - - return ((int) ((bitIndex + off) / 8)); - - } - - /** - * Return the offset within the byte in which the bit is coded of the bit - * (this is just the remainder <code>bitIndex % 8</code>). - * - * @param bitIndex - * The bit index into the byte[]. - * - * @return The offset of the bit in the appropriate byte. - */ - final protected int withinByteIndexForBit(final long bitIndex) { - - return (int) ((bitIndex + off) % 8); - - } - - /** - * Extract and return a bit coded flag. - * - * @param offset - * The offset in the buffer of the start of the byte[] sequence - * in which the bit coded flags are stored. - * @param index - * The index of the bit. - * - * @return The value of the bit. - */ - public boolean getBoolean(final long index) { - - if (index < 0 || index >= len) - throw new IndexOutOfBoundsException(); - - return (b.get(byteIndexForBit(index)) & (1 << withinByteIndexForBit(index))) != 0; - - } - -// // @todo override for mutation. -// public boolean set(final long index, final boolean value) { -// -// throw new UnsupportedOperationException(); -// -// } - -} Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/CSVReader.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/CSVReader.java 2010-09-30 11:47:53 UTC (rev 3690) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/CSVReader.java 2010-09-30 13:43:30 UTC (rev 3691) @@ -93,10 +93,8 @@ private final String name; - public String getName() { - - return name; - + public String getName() { + return name; } /** @@ -227,25 +225,29 @@ } /** - * Equal if the headers have the same data. + * Equal if the headers have the same name. */ - public boolean equals(Header o) { + @Override + public boolean equals(Object o) { + if (!(o instanceof Header)) + return false; + + Header h = (Header)o; - if(this==o) return true; - - return name.equals(o.name); - + return name.equals(h.name); } /** * Based on the header name. */ + @Override public int hashCode() { return name.hashCode(); } + @Override public String toString() { return name; @@ -289,7 +291,7 @@ * The header definitions (initially null). * * @see #readHeaders() - * @see #setHeaders(String[]) + * @see #setHeaders(Header[]) */ protected Header[] headers; @@ -324,54 +326,54 @@ } - public boolean setSkipCommentLines(boolean skipCommentLines) { +// public boolean setSkipCommentLines(boolean skipCommentLines) { +// +// boolean tmp = this.skipCommentLines; +// +// this.skipCommentLines = skipCommentLines; +// +// return tmp; +// +// } +// +// public boolean getSkipCommentLines() { +// +// return skipCommentLines; +// +// } - boolean tmp = this.skipCommentLines; +// public boolean setSkipBlankLines(boolean skipBlankLines) { +// +// boolean tmp = this.skipBlankLines; +// +// this.skipBlankLines = skipBlankLines; +// +// return tmp; +// +// } +// +// public boolean getSkipBlankLines() { +// +// return skipBlankLines; +// +// } - this.skipCommentLines = skipCommentLines; +// public boolean setTrimWhitespace(boolean trimWhitespace) { +// +// boolean tmp = this.trimWhitespace; +// +// this.trimWhitespace = trimWhitespace; +// +// return tmp; +// +// } +// +// public boolean getTrimWhitespace() { +// +// return trimWhitespace; +// +// } - return tmp; - - } - - public boolean getSkipCommentLines() { - - return skipCommentLines; - - } - - public boolean setSkipBlankLines(boolean skipBlankLines) { - - boolean tmp = this.skipBlankLines; - - this.skipBlankLines = skipBlankLines; - - return tmp; - - } - - public boolean getSkipBlankLines() { - - return skipBlankLines; - - } - - public boolean setTrimWhitespace(boolean trimWhitespace) { - - boolean tmp = this.trimWhitespace; - - this.trimWhitespace = trimWhitespace; - - return tmp; - - } - - public boolean getTrimWhitespace() { - - return trimWhitespace; - - } - /** * The #of milliseconds that the {@link CSVReader} should wait before * attempting to read another line from the source (when reading from @@ -505,8 +507,7 @@ } /** - * Trim whitespace and optional quotes from each value iff - * {@link #getTrimWhitespace()} is true. + * Trim whitespace and optional quotes. * * @param cols * The column values. @@ -651,7 +652,7 @@ */ public Header[] getHeaders() { - return headers.clone(); + return ((headers==null)? null : headers.clone()); } @@ -679,7 +680,7 @@ */ public void setHeader(int index,Header header) { - if (index < 0 || index > headers.length) + if (index < 0 || index >= headers.length) throw new IndexOutOfBoundsException(); if (header == null) @@ -688,7 +689,7 @@ headers[index] = header; } - + /** * Unsupported operation. */ Deleted: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/ChecksumError.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/ChecksumError.java 2010-09-30 11:47:53 UTC (rev 3690) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/ChecksumError.java 2010-09-30 13:43:30 UTC (rev 3691) @@ -1,46 +0,0 @@ -/** - -Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved. - -Contact: - SYSTAP, LLC - 4501 Tower Road - Greensboro, NC 27410 - lic...@bi... - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ -/* - * Created on Nov 5, 2006 - */ - -package com.bigdata.util; - -/** - * Exception thrown when the checksum field does not match the checksum computed - * for the data being read. This is a serious error and indicates bad logic - * and/or corrupt data. - * - * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ - */ -public class ChecksumError extends RuntimeException { - - private static final long serialVersionUID = -9067118459184074756L; - - public ChecksumError(String msg) { - super( msg ); - } - -} \ No newline at end of file Deleted: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/ClassLoaderUtil.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/ClassLoaderUtil.java 2010-09-30 11:47:53 UTC (rev 3690) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/ClassLoaderUtil.java 2010-09-30 13:43:30 UTC (rev 3691) @@ -1,328 +0,0 @@ -/* - -Copyright (C) SYSTAP, LLC 2006-2008. All rights reserved. - -Contact: - SYSTAP, LLC - 4501 Tower Road - Greensboro, NC 27410 - lic...@bi... - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -*/ - -package com.bigdata.util; - -import com.bigdata.util.config.LogUtil; - -import org.apache.log4j.Level; -import org.apache.log4j.Logger; - -import java.io.File; -import java.io.IOException; -import java.io.Serializable; - -import java.lang.reflect.Method; - -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLClassLoader; - -import java.rmi.Remote; -import java.rmi.RemoteException; -import java.rmi.activation.ActivationException; -import java.rmi.activation.ActivationID; - -import java.security.SecureClassLoader; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.StringTokenizer; - -import net.jini.io.MarshalledInstance; -import net.jini.loader.ClassAnnotation; - -/** - * This class provides useful utilities for creating and manipulating - * class loaders. Although it can be used for other purposes, it is - * generally intended for debugging. - */ -public class ClassLoaderUtil { - - /** Configure logger */ - private static Logger logger = - LogUtil.getLog4jLogger(ClassLoaderUtil.class); - - // Private constructor to prevent instantiation - private ClassLoaderUtil() { } - - /** - * Utility method that converts the components of a <code>String</code> - * representing a classpath into file <code>URL</code>(s). - * - * @param classpath <code>String</code> containing components separated - * by path separators that represent the components - * making up a classpath - * - * @return a <code>URL[]</code> where - * each element of the array corresponds to one of the components - * in the <code>classpath</code> parameter. The path components - * are (potentially) expanded via - * <code>File.getCanonicalFile()</code> before converting to a - * <code>URL</code> format. - * - * @throws java.net.MalformedURLException - * If the path cannot be parsed as a URL - * @throws java.net.IOException - * If an I/O error occurs, - * which is possible because the construction of - * the canonical pathname may require filesystem queries - */ - public static URL[] getClasspathURLs(String classpath) - throws IOException, MalformedURLException - { - StringTokenizer st = new StringTokenizer(classpath,File.pathSeparator); - URL[] urls = new URL[st.countTokens()]; - for (int i=0; st.hasMoreTokens(); i++) { - urls[i] = - new File(st.nextToken()).getCanonicalFile().toURI().toURL(); - } - return urls; - } - - /** - * Utility method that converts the components of a <code>String</code> - * representing a codebase into standard <code>URL</code>(s). - * - * @param codebase <code>String</code> containing components separated - * by spaces in which each component is in - * <code>URL</code> format. - * - * @return a <code>URL[]</code> where - * each element of the array corresponds to one of the components - * in the <code>codebase</code> parameter - * - * @throws java.net.MalformedURLException - */ - public static URL[] getCodebaseURLs(String codebase) - throws MalformedURLException - { - StringTokenizer st = new StringTokenizer(codebase); - URL[] urls = new URL[st.countTokens()]; - for (int i=0; st.hasMoreTokens(); i++) { - urls[i] = new URL(st.nextToken()); - } - return urls; - } - - /** - * Utility method that converts the components of a <code>String</code> - * representing a codebase or classpath into <code>URL</code>(s). - * - * @param importCodebase <code>String</code> assumed (in order) to be - * either - * 1) a space delimited set of <code>URL</code>(s) - * representing a codebase or - * 2) a <code>File.pathSeparator</code> delimited set - * of class paths. - * - * @return a <code>URL[]</code> where - * each element of the array corresponds to one of the components - * in the <code>importCodebase</code> parameter - * - * @throws java.net.MalformedURLException - * If the path cannot be parsed as a URL - * @throws java.net.IOException - * If an I/O error occurs, - * which is possible because the construction of - * the canonical pathname may require filesystem queries - */ - public static URL[] getImportCodebaseURLs(String importCodebase) - throws IOException, MalformedURLException - { - try { - return getCodebaseURLs(importCodebase); - } catch (MalformedURLException me) { - return getClasspathURLs(importCodebase); - } - } - - /** - * Utility method that retrieves the components making up the class loader - * delegation tree for the current context class loader and returns each - * in an <code>ArrayList</code>. - * - * @return an <code>ArrayList</code> instance in which each element of the - * list is one of the components making up the current delegation - * tree. - */ - private static ArrayList getContextClassLoaderTree() { - Thread curThread = Thread.currentThread(); - ClassLoader curClassLoader = curThread.getContextClassLoader(); - return getClassLoaderTree(curClassLoader); - } - - /** - * Utility method that retrieves the components making up the class loader - * delegation tree for the given <code>classloader</code> parameter and - * returns them via an <code>ArrayList</code>. - * - * @param classloader <code>ClassLoader</code> instance whose delegation - * tree is to be retrieved and returned - * - * @return an <code>ArrayList</code> instance in which each element of the - * list is one of the components making up the delegation tree - * of the given class loader. - */ - private static ArrayList getClassLoaderTree(ClassLoader classloader) { - ArrayList loaderList = new ArrayList(); - while(classloader != null) { - loaderList.add(classloader); - classloader = classloader.getParent(); - } - loaderList.add(null); //Append boot classloader - Collections.reverse(loaderList); - return loaderList; - } - - /** - * Utility method that displays the class loader delegation tree for - * the current context class loader. For each class loader in the tree, - * this method displays the locations from which that class loader - * will retrieve and load requested classes. - * <p> - * This method can be useful when debugging problems related to the - * receipt of exceptions such as <code>ClassNotFoundException</code>. - */ - public static void displayContextClassLoaderTree() { - Thread curThread = Thread.currentThread(); - ClassLoader curClassLoader = curThread.getContextClassLoader(); - displayClassLoaderTree(curClassLoader); - } - - /** - * Utility method that displays the class loader delegation tree for - * the given class loader. For each class loader in the tree, this - * method displays the locations from which that class loader will - * retrieve and load requested classes. - * <p> - * This method can be useful when debugging problems related to the - * receipt of exceptions such as <code>ClassNotFoundException</code>. - * - * Note that although this class' logger level is used to determine - * whether or not to display any information at all, the output is - * actually displayed using System.out.println. This is done to - * produce more readable output than the logger might produce. - * - * @param description descriptive <code>String</code> that, if - * non-<code>null</code>, will be logged prior to - * displaying the information about the - * <code>classloader</code>. - * - * @param classloader <code>ClassLoader</code> instance whose delegation - * tree is to be displayed. - */ - public static void displayClassLoaderTree(ClassLoader classloader) { - displayClassLoaderTree(null, classloader); - } - - public static void displayClassLoaderTree(String description, - ClassLoader classloader) - { - if( logger.isEnabledFor(Level.DEBUG) ) { - if(description != null) { - logger.log(Level.DEBUG, description); - } - - ArrayList loaderList = getClassLoaderTree(classloader); - System.out.println("ClassLoader Tree has " - + loaderList.size() + " levels"); - System.out.println(" cl0 -- Boot ClassLoader "); - ClassLoader curClassLoader = null; - for(int i=1; i < loaderList.size(); i++) { - System.out.println(" |"); - curClassLoader = (ClassLoader)loaderList.get(i); - System.out.print(" cl"+i+" -- ClassLoader " - +curClassLoader+": "); - if(curClassLoader instanceof URLClassLoader) { - URL[] urls = ((URLClassLoader)(curClassLoader)).getURLs(); - if(urls != null) { - System.out.print(urls[0]); - for(int j=1;j<urls.length;j++){ - System.out.print(", "+urls[j]); - } - } else {//urls == null - System.out.print("null search path"); - } - } else { - if(curClassLoader instanceof SecureClassLoader) { - System.out.print("is instance of SecureClassLoader"); - } else { - System.out.print("is unknown ClassLoader type"); - } - } - System.out.println(""); - } - System.out.println(""); - } - } - - /** - * Handles a <i>class loader mismatch</i> between the given - * <code>Serializable</code> object and the given <code>Class</code> - * type. - * - * If the class name of the given <code>obj</code> parameter - * is the same as the name of <code>classType</code>, but - * <code>obj</code> is not an instance of <code>classType</code>, - * then the difference may be due to unequal class loaders for the - * two parameters; which is referred to as a <i>class loader mismatch</i>. - * When such a mismatch occurs, the <code>instanceof</code> operator - * will return <code>false</code> and attempts to cast the given - * <code>obj</code> to the given <code>classType</code> will result - * in a <code>ClassCastException</code>. - * - * To address the situation just described, this method attempts to - * "reload" the given <code>obj</code>, using the <code>ClassLoader</code> - * of <code>classType</code>. This is accomplished by first - * marshalling and then unmarshalling the given <code>obj</code>, - * while the <i>current context class loader</i> is set to the - * <code>ClassLoader</code> of <code>classType</code>. - * - * Upon success, the newly loaded object is returned; which can then - * be successfully cast to the given <code>classType</code>. If the - * reload operation fails, <code>null</code> is returned. - */ - public static Serializable instanceOf(Serializable obj, Class classType) { - - if( classType.isInstance(obj) ) return obj; - - Class objClass = obj.getClass(); - ClassLoader classTypeCl = classType.getClassLoader(); - //marshall-and-unmarshal using the class type's classloader - try { - MarshalledInstance mInst = new MarshalledInstance(obj); - Serializable newObj = - (Serializable)(mInst.get(classTypeCl, false, null, null)); - if( classType.isInstance(newObj) ) return newObj; - - } catch(Throwable t) { - return null; - } - return null; - } - -} Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/EntryUtil.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/EntryUtil.java 2010-09-30 11:47:53 UTC (rev 3690) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/util/EntryUtil.java 2010-09-30 13:43:30 UTC (rev 3691) @@ -75,7 +75,7 @@ * type. If no element in the array is an instance of the given * type, then <code>null</code> is returned. */ - public static <T> T getEntryByType(Entry[] attrs, Class<T> type) + public static <T extends Entry> T getEntryByType(Entry[] attrs, Class<T> type) { if( (attrs == null) || (type == null) ) return null; for(int i=0; i<attrs.length; i++) { @@ -85,144 +85,144 @@ return null; }//end getEntryByType - /** - * Examines the given array of entries and returns an array of all - * elements that are an instance of the given class type. If no element in - * the array is an instance of the given type, then <code>null</code> is - * returned. - */ - public static <T> T[] getEntriesByType(Entry[] attrs, Class<T> type) - { - if( (attrs == null) || (type == null) ) return null; - ArrayList<T> matches = null; - for(int i=0; i<attrs.length; i++) { - if( !( type.isInstance(attrs[i]) ) ) continue; - if ( matches == null ) - matches = new ArrayList<T>(); - matches.add( (T)attrs[i] ); - }//end loop +// /** +// * Examines the given array of entries and returns an array of all +// * elements that are an instance of the given class type. If no element in +// * the array is an instance of the given type, then <code>null</code> is +// * returned. +// */ +// public static <T> T[] getEntriesByType(Entry[] attrs, Class<T> type) +// { +// if( (attrs == null) || (type == null) ) return null; +// ArrayList<T> matches = null; +// for(int i=0; i<attrs.length; i++) { +// if( !( type.isInstance(attrs[i]) ) ) continue; +// if ( matches == null ) +// matches = new ArrayList<T>(); +// matches.add( (T)attrs[i] ); +// }//end loop +// +// if ( matches == null ) return null; +// +// return matches.toArray +// ( (T[])java.lang.reflect.Array.newInstance +// ( type, matches.size() ) ); +// } - if ( matches == null ) return null; +// /** +// * Examines the given array of entries and returns a list of all +// * elements that are an instance of the given class type. If no element +// * in the array is an instance of the given type, then an empty list is +// * returned. +// */ +// public static <T> List<T> getEntryListByType(final Entry[] attrs, +// final Class<T> type) +// { +// if(attrs == null) { +// throw new NullPointerException("null attrs array"); +// } +// if(type == null) { +// throw new NullPointerException("null type"); +// } +// final List<T> matches = new ArrayList<T>(); +// for (final Entry attr : attrs) { +// if(type.isInstance(attr)){ +// matches.add(type.cast(attr)); +// } +// } +// return matches; +// } - return matches.toArray - ( (T[])java.lang.reflect.Array.newInstance - ( type, matches.size() ) ); - } +// /** +// * Examines the given array of entries and returns a list of all +// * elements that match the given <code>template</code>; where +// * the matching semantics are defined by the template-matching +// * semantics specified in the Lookup Service Specification. If no +// * element in the given array matches the template, then an empty +// * list is returned. +// */ +// public static <T extends Entry> List<T> getEntryList(final Entry[] attrs, +// final T template) +// { +// final List<T> matches = new ArrayList<T>(); +// for (final Entry attr : attrs) { +// if(LookupAttributes.matches(template, attr)){ +// matches.add((T) attr); +// } +// } +// return matches; +// } - /** - * Examines the given array of ... [truncated message content] |
From: <sgo...@us...> - 2010-10-01 19:03:31
|
Revision: 3716 http://bigdata.svn.sourceforge.net/bigdata/?rev=3716&view=rev Author: sgossard Date: 2010-10-01 19:03:22 +0000 (Fri, 01 Oct 2010) Log Message: ----------- [maven_scaleout] : Broke all direct dependency cycles with package 'com.bigdata.concurrent'. Removed old NonBlockingLockManager class and moved logic from NonBlockingLockManagerWithNewDesign that updated journal related write counters into WriteExecutorService.MyLockManager subclass already in journal package. Modified Paths: -------------- branches/maven_scaleout/bigdata-core/bigdata-perf/bsbm/src/resources/logging/log4j.properties branches/maven_scaleout/bigdata-core/bigdata-perf/btc/src/resources/logging/log4j.properties branches/maven_scaleout/bigdata-core/bigdata-perf/lubm/src/resources/logging/log4j.properties branches/maven_scaleout/bigdata-core/bigdata-perf/uniprot/src/resources/logging/log4j.properties branches/maven_scaleout/bigdata-core/src/main/deploy/legacy/bigdata-resources/logging/log4j-perf-tests.properties branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/logging/log4j.properties branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/concurrent/NonBlockingLockManagerWithNewDesign.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/journal/AbstractTask.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/journal/ConcurrencyManager.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/journal/WriteExecutorService.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/concurrent/AbstractStressTestNonBlockingLockManager.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/concurrent/TestAll.java Removed Paths: ------------- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/concurrent/NonBlockingLockManager.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/concurrent/TestNonBlockingLockManager.java Modified: branches/maven_scaleout/bigdata-core/bigdata-perf/bsbm/src/resources/logging/log4j.properties =================================================================== --- branches/maven_scaleout/bigdata-core/bigdata-perf/bsbm/src/resources/logging/log4j.properties 2010-10-01 16:48:09 UTC (rev 3715) +++ branches/maven_scaleout/bigdata-core/bigdata-perf/bsbm/src/resources/logging/log4j.properties 2010-10-01 19:03:22 UTC (rev 3716) @@ -68,7 +68,6 @@ #log4j.logger.com.bigdata.service.AbstractTransactionService=INFO #log4j.logger.com.bigdata.journal.AbstractLocalTransactionManager=INFO #log4j.logger.com.bigdata.concurrent.TxDag=WARN -#log4j.logger.com.bigdata.concurrent.NonBlockingLockManager=WARN #log4j.logger.com.bigdata.concurrent.TestNonBlockingLockManager=INFO #log4j.logger.com.bigdata.concurrent.AbstractStressTestNonBlockingLockManager=INFO #log4j.logger.com.bigdata.concurrent.LockManager=INFO Modified: branches/maven_scaleout/bigdata-core/bigdata-perf/btc/src/resources/logging/log4j.properties =================================================================== --- branches/maven_scaleout/bigdata-core/bigdata-perf/btc/src/resources/logging/log4j.properties 2010-10-01 16:48:09 UTC (rev 3715) +++ branches/maven_scaleout/bigdata-core/bigdata-perf/btc/src/resources/logging/log4j.properties 2010-10-01 19:03:22 UTC (rev 3716) @@ -67,7 +67,6 @@ #log4j.logger.com.bigdata.service.AbstractTransactionService=INFO #log4j.logger.com.bigdata.journal.AbstractLocalTransactionManager=INFO #log4j.logger.com.bigdata.concurrent.TxDag=WARN -#log4j.logger.com.bigdata.concurrent.NonBlockingLockManager=WARN #log4j.logger.com.bigdata.concurrent.TestNonBlockingLockManager=INFO #log4j.logger.com.bigdata.concurrent.AbstractStressTestNonBlockingLockManager=INFO #log4j.logger.com.bigdata.concurrent.LockManager=INFO Modified: branches/maven_scaleout/bigdata-core/bigdata-perf/lubm/src/resources/logging/log4j.properties =================================================================== --- branches/maven_scaleout/bigdata-core/bigdata-perf/lubm/src/resources/logging/log4j.properties 2010-10-01 16:48:09 UTC (rev 3715) +++ branches/maven_scaleout/bigdata-core/bigdata-perf/lubm/src/resources/logging/log4j.properties 2010-10-01 19:03:22 UTC (rev 3716) @@ -57,7 +57,6 @@ #log4j.logger.com.bigdata.service.AbstractTransactionService=INFO #log4j.logger.com.bigdata.journal.AbstractLocalTransactionManager=INFO log4j.logger.com.bigdata.concurrent.TxDag=WARN -log4j.logger.com.bigdata.concurrent.NonBlockingLockManager=WARN log4j.logger.com.bigdata.concurrent.TestNonBlockingLockManager=INFO log4j.logger.com.bigdata.concurrent.AbstractStressTestNonBlockingLockManager=INFO #log4j.logger.com.bigdata.concurrent.LockManager=INFO Modified: branches/maven_scaleout/bigdata-core/bigdata-perf/uniprot/src/resources/logging/log4j.properties =================================================================== --- branches/maven_scaleout/bigdata-core/bigdata-perf/uniprot/src/resources/logging/log4j.properties 2010-10-01 16:48:09 UTC (rev 3715) +++ branches/maven_scaleout/bigdata-core/bigdata-perf/uniprot/src/resources/logging/log4j.properties 2010-10-01 19:03:22 UTC (rev 3716) @@ -67,7 +67,6 @@ #log4j.logger.com.bigdata.service.AbstractTransactionService=INFO #log4j.logger.com.bigdata.journal.AbstractLocalTransactionManager=INFO #log4j.logger.com.bigdata.concurrent.TxDag=WARN -#log4j.logger.com.bigdata.concurrent.NonBlockingLockManager=WARN #log4j.logger.com.bigdata.concurrent.TestNonBlockingLockManager=INFO #log4j.logger.com.bigdata.concurrent.AbstractStressTestNonBlockingLockManager=INFO #log4j.logger.com.bigdata.concurrent.LockManager=INFO Modified: branches/maven_scaleout/bigdata-core/src/main/deploy/legacy/bigdata-resources/logging/log4j-perf-tests.properties =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/deploy/legacy/bigdata-resources/logging/log4j-perf-tests.properties 2010-10-01 16:48:09 UTC (rev 3715) +++ branches/maven_scaleout/bigdata-core/src/main/deploy/legacy/bigdata-resources/logging/log4j-perf-tests.properties 2010-10-01 19:03:22 UTC (rev 3716) @@ -52,7 +52,6 @@ #log4j.logger.com.bigdata.service.AbstractTransactionService=INFO #log4j.logger.com.bigdata.journal.AbstractLocalTransactionManager=INFO log4j.logger.com.bigdata.concurrent.TxDag=WARN -log4j.logger.com.bigdata.concurrent.NonBlockingLockManager=WARN log4j.logger.com.bigdata.concurrent.TestNonBlockingLockManager=INFO log4j.logger.com.bigdata.concurrent.AbstractStressTestNonBlockingLockManager=INFO #log4j.logger.com.bigdata.concurrent.LockManager=INFO Modified: branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/logging/log4j.properties =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/logging/log4j.properties 2010-10-01 16:48:09 UTC (rev 3715) +++ branches/maven_scaleout/bigdata-core/src/main/deploy/var/config/logging/log4j.properties 2010-10-01 19:03:22 UTC (rev 3716) @@ -56,7 +56,6 @@ #log4j.logger.com.bigdata.service.AbstractTransactionService=INFO #log4j.logger.com.bigdata.journal.AbstractLocalTransactionManager=INFO log4j.logger.com.bigdata.concurrent.TxDag=WARN -log4j.logger.com.bigdata.concurrent.NonBlockingLockManager=WARN log4j.logger.com.bigdata.concurrent.TestNonBlockingLockManager=INFO log4j.logger.com.bigdata.concurrent.AbstractStressTestNonBlockingLockManager=INFO #log4j.logger.com.bigdata.concurrent.LockManager=INFO Deleted: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/concurrent/NonBlockingLockManager.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/concurrent/NonBlockingLockManager.java 2010-10-01 16:48:09 UTC (rev 3715) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/concurrent/NonBlockingLockManager.java 2010-10-01 19:03:22 UTC (rev 3716) @@ -1,2472 +0,0 @@ -/** - - Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved. - - Contact: - SYSTAP, LLC - 4501 Tower Road - Greensboro, NC 27410 - lic...@bi... - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ -/* - * Created on Oct 3, 2007 - */ - -package com.bigdata.concurrent; - -import static com.bigdata.concurrent.NonBlockingLockManager.RunState.Halted; -import static com.bigdata.concurrent.NonBlockingLockManager.RunState.Running; -import static com.bigdata.concurrent.NonBlockingLockManager.RunState.Shutdown; -import static com.bigdata.concurrent.NonBlockingLockManager.RunState.ShutdownNow; -import static com.bigdata.concurrent.NonBlockingLockManager.RunState.Starting; - -import java.lang.ref.WeakReference; -import java.util.Arrays; -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.Map; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Callable; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.FutureTask; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.ReentrantLock; - -import org.apache.log4j.Logger; - -import com.bigdata.cache.ConcurrentWeakValueCacheWithTimeout; -import com.bigdata.counters.CounterSet; -import com.bigdata.counters.Instrument; -import com.bigdata.journal.AbstractTask; -import com.bigdata.util.concurrent.DaemonThreadFactory; -import com.bigdata.util.concurrent.ThreadPoolExecutorStatisticsTask; -import com.bigdata.util.concurrent.WriteTaskCounters; - -/** - * This class coordinates a schedule among concurrent operations requiring - * exclusive access to shared resources. Whenever possible, the result is a - * concurrent schedule - that is, operations having non-overlapping lock - * requirements run concurrently while operations that have lock contentions are - * queued behind operations that currently have locks on the relevant resources. - * A {@link ResourceQueue} is created for each resource and used to block - * operations that are awaiting a lock. When locks are not being pre-declared, a - * {@link TxDag WAITS_FOR} graph is additionally used to detect deadlocks. - * <p> - * This implementation uses a single {@link AcceptTask} thread to accept tasks, - * update the requests in the {@link ResourceQueue}s, and in general perform - * housekeeping for the internal state. Tasks submitted to this class ARE NOT - * bound to a worker thread until they are executed by the delegate - * {@link Executor}. - * - * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ - * - * @param R - * The type of the object that identifies a resource for the purposes - * of the locking system. This is typically the name of an index. - * - * @todo a fair option? What constraints or freedoms would it provide? - * - * @todo a {@link SynchronousQueue} for the {@link #acceptedTasks}? - * - * @todo a {@link SynchronousQueue} for the writeService workQueue? - * - * FIXME In order to support 2PL we need to decouple the {@link LockFutureTask} - * from the transaction with which it is associated. Otherwise each - * {@link #submit(Comparable[], Callable)} will look like a new transaction (2PL - * is impossible unless you can execute multiple tasks for the same - * transaction). - * <p> - * Perhaps this would be easier if we did not delegate the task to an - * {@link Executor} since that does not really support the 2PL pattern. - * - * @todo Support escalation of operation priority based on time and scheduling - * of higher priority operations. the latter is done by queueing lock - * requests in front of pending requests for each resource on which an - * operation attempt to gain a lock. The former is just a dynamic - * adjustment of the position of the operation in the resource queue where - * it is awaiting a lock (an operation never awaits more than one lock at - * a time). This facility could be used to give priority to distributed - * transactions over local unisolated operations and to priviledge certain - * operations that have low latency requirements. This is not quite a - * "real-time" guarentee since the VM is not (normally) providing - * real-time guarentees and since we are not otherwise attempting to - * ensure anything except lower latency when compared to other operations - * awaiting their own locks. - */ -public class NonBlockingLockManager</* T, */R extends Comparable<R>> { - - final protected static Logger log = Logger - .getLogger(NonBlockingLockManager.class); - - final protected static boolean INFO = log.isInfoEnabled(); - final protected static boolean DEBUG = log.isDebugEnabled(); - - /** - * Each resource that can be locked has an associated {@link ResourceQueue}. - * <p> - * Note: This is a concurrent collection since new resources may be added - * while concurrent operations resolve resources to their queues. Stale - * {@link ResourceQueue}s are purged after they become only weakly - * reachable. - * - * @todo reconsider the timeout. it is set to one LBS period right now. - */ - final private ConcurrentWeakValueCacheWithTimeout<R, ResourceQueue<LockFutureTask<? extends Object>>> resourceQueues = new ConcurrentWeakValueCacheWithTimeout<R, ResourceQueue<LockFutureTask<? extends Object>>>( - 1000/* nresources */, TimeUnit.SECONDS.toNanos(60)); - - /** - * Release all locks held by the {@link LockFutureTask} currently holding a - * lock on the specified resource. - * - * @param resource[] - * The declared locks for the task. - * - * FIXME This is an integration hack for {@link AbstractTask}. - * {@link AbstractTask} needs to be able to release the locks as soon as the - * work of an unisolated task is done (when it is waiting for a group - * commit) so that other tasks can gain access to the same indices and make - * it into the same group commit. It is using this method to obtain the - * {@link LockFutureTask} and then release its locks. - * <p> - * {@link AbstractTask} really needs a refactor. - */ - public final void releaseLocksForTask(final R[] resource) { - - if (resource == null) - throw new IllegalArgumentException(); - - if(resource.length == 0) { - - // No declared locks. - return; - - } - - lock.lock(); - try { - - LockFutureTask<? extends Object> task = null; - - for (R r : resource) { - - final ResourceQueue<LockFutureTask<? extends Object>> resourceQueue = resourceQueues - .get(r); - - if (task == null) { - - /* - * find the task by checking the resource queue for any of - * its declared locks. - */ - task = resourceQueue.queue.peek(); - - if (task == null) - throw new IllegalArgumentException( - "Task does not hold declared lock: " + r); - - } else { - - /* - * verify that the task holds the rest of its declared - * locks. - */ - if (task != resourceQueue.queue.peek()) { - - throw new IllegalArgumentException( - "Task does not hold declared lock: " + r); - - } - - } - - } - - if(task == null) { - - throw new AssertionError(); - - } - - /* - * At this point we have verified that there is a task which holds - * all of the locks specified by the caller. - */ - - /* - * The task will be running since that is the presumption for the - * context in which this method is invoked within AbstractTask. - */ - final boolean waiting = false; - - // release the locks for the task. - releaseLocks(task, waiting); - - } finally { - - lock.unlock(); - - } - - } - - /** - * True iff locks MUST be predeclared by the operation - this is a special - * case of 2PL (two-phrase locking) that allows significant optimizations - * and avoids the possibility of deadlock altogether. - */ - final private boolean predeclareLocks; - - /** - * When true, the resources in a lock request are sorted before the lock - * requests are made to the various resource queues. This option is ONLY - * turned off for testing purposes as it ALWAYS reduces the chance of - * deadlocks and eliminates it entirely when locks are also predeclared. - */ - final private boolean sortLockRequests; - - /** - * Used to track dependencies among transactions. - */ - final protected TxDag waitsFor; - - /** - * Tasks holding their locks are submitted to this service for execution. - */ - final protected Executor delegate; - - /** - * Run states for the {@link NonBlockingLockManager}. - * - * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ - */ - static public enum RunState { - - /** - * During startup. Tasks are NOT accepted. - */ - Starting(0), - - /** - * While running (aka open). Tasks are accepted and submitted for - * execution once they hold their locks. - */ - Running(1), - - /** - * When shutting down normally. New tasks are not accepted but - * {@link Future}s are still monitored for completion and waiting tasks - * will eventually be granted their locks and execute on the delegate. - */ - Shutdown(2), - - /** - * When shutting down immediately. New tasks are not accepted, tasks - * waiting for their locks are cancelled (they will not execute) and - * {@link Future}s for running tasks are cancelled (they are - * interrupted). - */ - ShutdownNow(3), - - /** - * When halted. New tasks are not accepted. No tasks are waiting. Any - * {@link Future}s were cancelled. - */ - Halted(4); - - private RunState(int val) { - - this.val = val; - - } - - final private int val; - - public int value() { - - return val; - - } - - public boolean isTransitionLegal(final RunState newval) { - - if (this == Starting) { - - if (newval == Running) - return true; - - if (newval == Halted) - return true; - - } else if (this == Running) { - - if (newval == Shutdown) - return true; - - if (newval == ShutdownNow) - return true; - - } else if (this == Shutdown) { - - if (newval == ShutdownNow) - return true; - - if (newval == Halted) - return true; - - } else if (this == ShutdownNow) { - - if (newval == Halted) - return true; - - } - - return false; - - } - - } - - /* - * counters - */ - - synchronized public CounterSet getCounters() { - - if (root == null) { - - root = new CounterSet(); - - root.addCounter("naccepted", new Instrument<Long>() { - public void sample() { - setValue(counters.naccepted); - } - }); - - root.addCounter("nrejected", new Instrument<Long>() { - public void sample() { - setValue(counters.nrejected); - } - }); - - root.addCounter("nstarted", new Instrument<Long>() { - public void sample() { - setValue(counters.nstarted); - } - }); - - root.addCounter("nended", new Instrument<Long>() { - public void sample() { - setValue(counters.nended); - } - }); - - root.addCounter("ncancel", new Instrument<Long>() { - public void sample() { - setValue(counters.ncancel); - } - }); - - root.addCounter("nerror", new Instrument<Long>() { - public void sample() { - setValue(counters.nerror); - } - }); - - root.addCounter("ndeadlock", new Instrument<Long>() { - public void sample() { - setValue(counters.ndeadlock); - } - }); - - root.addCounter("ntimeout", new Instrument<Long>() { - public void sample() { - setValue(counters.ntimeout); - } - }); - - // Note: #that are waiting for their locks. - root.addCounter("nwaiting", new Instrument<Integer>() { - public void sample() { - setValue(counters.nwaiting); - } - }); - - // Note: #that have acquired locks are executing concurrently. - root.addCounter("nrunning", new Instrument<Integer>() { - public void sample() { - setValue(counters.nrunning); - } - }); - - // the maximum observed value for [nrunning]. - root.addCounter("maxRunning", new Instrument<Integer>() { - public void sample() { - setValue(counters.maxRunning); - } - }); - - // #of resource queues - root.addCounter("nresourceQueues", new Instrument<Integer>() { - public void sample() { - setValue(resourceQueues.size()); - } - }); - - root.addCounter("runState", new Instrument<String>() { - public void sample() { - setValue(runState.toString()); - } - }); - - /* - * Displays the #of tasks waiting on each resource queue for the - * lock (does not count the task at the head of the queue). - * - * @todo this could also be handled by dynamic reattachment of the - * counters. - */ - root.addCounter("queues", new Instrument<String>() { - public void sample() { - final Iterator<Map.Entry<R, WeakReference<ResourceQueue<LockFutureTask<? extends Object>>>>> itr = resourceQueues - .entryIterator(); - final LinkedList<ResourceQueueSize> list = new LinkedList<ResourceQueueSize>(); - while (itr.hasNext()) { - final Map.Entry<R, WeakReference<ResourceQueue<LockFutureTask<? extends Object>>>> entry = itr - .next(); - final WeakReference<ResourceQueue<LockFutureTask<? extends Object>>> queueRef = entry - .getValue(); - final ResourceQueue<LockFutureTask<? extends Object>> queue = queueRef - .get(); - if (queue == null) - continue; - list.add(new ResourceQueueSize(queue)); - } - final Object[] a = list.toArray(); - Arrays.sort(a); - final StringBuilder sb = new StringBuilder(); - for (Object t : a) { - sb.append(t.toString()); - sb.append(" "); - } - setValue(sb.toString()); - } - }); - - } - - return root; - - } - - private CounterSet root; - - /** - * Helper class pairs up the resource with its sampled queue size and allows - * ordering of the samples. E.g., by size or by resource. - * - * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ - */ - private class ResourceQueueSize implements Comparable<ResourceQueueSize> { - final R resource; - int size; - public ResourceQueueSize(ResourceQueue<LockFutureTask<? extends Object>> queue){ - resource = queue.getResource(); - size = queue.getQueueSize(); - } - public int compareTo(ResourceQueueSize arg0) { - // resource name order. -// return resource.compareTo(arg0.resource); - // descending queue size order. - return arg0.size - size; - } - public String toString() { - return "(" + resource + "," + size + ")"; - } - } - - /** - * Counters for the {@link NonBlockingLockManager}. - * - * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ - */ - protected static class Counters { - - /** - * The #of tasks that were accepted by the service (running total). - */ - public long naccepted; - - /** - * The #of tasks that were rejected by the service (running total). - */ - public long nrejected; - - /** - * The #of tasks that have been started on the delegate {@link Executor} - * (running total). - */ - public long nstarted; - - /** - * The #of tasks that whose execution on the delegate {@link Executor} - * is complete (either by normal completion or by error, but only for - * tasks which were executed on the delegate) (running total). - */ - public long nended; - - /** - * The #of tasks that were cancelled (running total). - */ - public long ncancel; - - /** - * The #of tasks whose exception was set (running total). - */ - public long nerror; - - /** - * The #of tasks that deadlocked when they attempted to acquire their - * locks (running total). Note that a task MAY retry lock acquisition - * and this counter will be incremented each time it does so and then - * deadlocks. - */ - public long ndeadlock; - - /** - * The #of tasks that timed out when they attempted to acquire their - * locks (running total). Note that a task MAY retry lock acquisition - * and this counter will be incremented each time it does so and then - * times out. - */ - public long ntimeout; - - /** - * #of tasks that are currently waiting on locks. This is the effective - * queue length of the {@link NonBlockingLockManager}. To get the - * actual queue length you need to add this to the length of the queue - * for the delegate {@link Executor}. - * - * FIXME This counter can be off since a task does not "know" when it is - * placed onto the [waitingTasks] queue and therefore can not decrement - * the counter conditionally if the task is cancelled or if an exception - * is set. Right now the counter is only decremented if the task begins - * to execute. This is a problem since this is the primary indication of - * the total size of the {@link NonBlockingLockManager} as a queue - * feeding its delegate {@link Executor}. - */ - public int nwaiting; - - /** - * #of tasks that have acquired their locks and are concurrently - * executing. This is the true measure of concurrency. - */ - public int nrunning; - - /** - * The maximum observed value of {@link #nrunning}. - */ - public int maxRunning; - - } - - /** - * Counters for various things. - */ - final Counters counters = new Counters(); - - /** - * Create a lock manager. No concurrency limit imposed when - * <i>predeclareLocks</i> is <code>true</code> as deadlocks are - * impossible and we do not maintain a WAITS_FOR graph. - * - * @param maxConcurrency - * The maximum multi-programming level (ignored if - * <i>predeclareLocks</i> is <code>true</code>). - * @param predeclareLocks - * When <code>true</code>, operations MUST declare all locks - * before they begin to execute. This makes possible several - * efficiencies and by sorting the resources in each lock request - * into a common order we are able to avoid deadlocks entirely. - * @param delegate - * The service on which the tasks will be executed. - * <p> - * Note: The <i>delegate</i> MUST NOT use a bounded queue or - * cause tasks to be run in the caller's thread. The use of a - * {@link SynchronousQueue} or an unbounded - * {@link LinkedBlockingQueue} for the <i>delegate</i>'s - * workQueue are both acceptable. - * <p> - * Note: If {@link Executor#execute(Runnable)} blocks for the - * <i>delegate</i> then the {@link AcceptTask} will also block - * and this class will be non-responsive until the <i>delegate</i> - * has accepted each [waitingTask] for execution. Some - * {@link Executor}s can cause the task to be run in the - * caller's thread, which would be the {@link AcceptTask} itself - * and which also has the effect of causing this class to be - * non-responsive until the task is complete. - */ - public NonBlockingLockManager(final int maxConcurrency, - final boolean predeclareLocks, final Executor delegate) { - - this(maxConcurrency, predeclareLocks, true/* sortLockRequests */, - delegate); - - } - - /** - * Create a lock manager. No concurrency limit imposed when - * <i>predeclareLocks</i> is <code>true</code> as deadlocks are - * impossible and we do not maintain a WAITS_FOR graph. - * - * @param maxConcurrency - * The maximum multi-programming level (ignored if - * <i>predeclareLocks</i> is <code>true</code>). - * @param predeclareLocks - * When <code>true</code>, operations MUST declare all locks - * before they begin to execute. This makes possible several - * efficiencies and by sorting the resources in each lock request - * into a common order we are able to avoid deadlocks entirely. - * @param sortLockRequests - * This option indicates whether or not the resources in a lock - * request will be sorted before attempting to acquire the locks - * for those resources. Normally <code>true</code> this option - * MAY be disabled for testing purposes. It is an error to - * disable this option if <i>predeclareLocks</i> is - * <code>false</code>. - * @param delegate - * The service on which the tasks will be executed. - * <p> - * Note: The <i>delegate</i> MUST NOT use a bounded queue or - * cause tasks to be run in the caller's thread. The use of a - * {@link SynchronousQueue} or an unbounded - * {@link LinkedBlockingQueue} for the <i>delegate</i>'s - * workQueue are both acceptable. - * <p> - * Note: If {@link Executor#execute(Runnable)} blocks for the - * <i>delegate</i> then the {@link AcceptTask} will also block - * and this class will be non-responsive until the <i>delegate</i> - * has accepted each [waitingTask] for execution. Some - * {@link Executor}s can cause the task to be run in the - * caller's thread, which would be the {@link AcceptTask} itself - * and which also has the effect of causing this class to be - * non-responsive until the task is complete. - */ - NonBlockingLockManager(final int maxConcurrency, - final boolean predeclareLocks, final boolean sortLockRequests, - final Executor delegate) { - - if (maxConcurrency < 2 && !predeclareLocks) { - - throw new IllegalArgumentException( - "maxConcurrency: must be 2+ unless you are predeclaring locks, not " - + maxConcurrency); - - } - - if (predeclareLocks && !sortLockRequests) { - - /* - * This is required since we do not maintain TxDag when locks are - * predeclare and therefore can not detect deadlocks. Sorting with - * predeclared locks avoids the possibility of deadlocks so we do - * not need the TxDag (effectively, it means that all locks that can - * be requested by an operation are sorted since they are - * predeclared and acquired in one go). - */ - - throw new IllegalArgumentException( - "Sorting of lock requests MUST be enabled when locks are being predeclared."); - - } - - if (delegate == null) - throw new IllegalArgumentException(); - - this.predeclareLocks = predeclareLocks; - - this.sortLockRequests = sortLockRequests; - - if (predeclareLocks) { - - /* - * Note: waitsFor is NOT required if we will acquire all locks at - * once for a given operation since we can simply sort the lock - * requests for each operation into a common order, thereby making - * deadlock impossible! - * - * Note: waitsFor is also NOT required if we are using only a single - * threaded system. - * - * Note: if you allocate waitsFor here anyway then you can measure - * the cost of deadlock detection. As far as I can tell it is - * essentially zero when locks are predeclared. - */ - - waitsFor = null; - - // waitsFor = new TxDag(maxConcurrency); - - } else { - - /* - * Construct the directed graph used to detect deadlock cycles. - */ - - waitsFor = new TxDag(maxConcurrency); - - } - - this.delegate = delegate; - - // start service. - service.submit(new AcceptTask()); - - // change the run state. - lock.lock(); - try { - - setRunState(RunState.Running); - - } finally { - - lock.unlock(); - - } - - } - - /** - * {@link FutureTask} which executes once it holds its locks. - * - * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ - * - * @param <T> - * The generic type of the outcome for the {@link Future}. - */ - protected class LockFutureTask<T> extends FutureTask<T> { - - private final R[] resource; - - private final long lockTimeout; - - private final int maxLockTries; - - /** - * Incremented each time a deadlock is detected. We will not retry if - * {@link #maxLockTries} is exceeded. - */ - private int ntries = 0; - - /** - * The timestamp in nanoseconds when this task was accepted. This is - * used to decide whether the {@link #lockTimeout} has expired. - */ - final private long acceptTime = System.nanoTime(); - - /** - * The set of {@link ResourceQueue}s for which this task owns a lock - * (is a member of the granted group) (NOT THREAD SAFE). - * <p> - * Note: This collection is required in order for the - * {@link ResourceQueue}s for which the task has asserted a lock - * request to remain strongly reachable. Without such hard references - * the {@link ResourceQueue}s would be asynchronously cleared from the - * {@link NonBlockingLockManager#resourceQueues} collection by the - * garbage collector. - */ - private final LinkedHashSet<ResourceQueue<LockFutureTask<? extends Object>>> lockedResources = new LinkedHashSet<ResourceQueue<LockFutureTask<? extends Object>>>(); - - /** - * True if the {@link #lockTimeout} has expired when measured against - * <i>now</i>. - */ - protected boolean isTimeout() { - - return (System.nanoTime() - acceptTime) >= lockTimeout; - - } - - private final Object task; - - public String toString() { - - return super.toString() + // - "{resources=" + Arrays.toString(resource) + // - ", done=" + isDone() + // - ", cancelled=" + isCancelled() + // - ", ntries=" + ntries + - "}"; - - } - - public LockFutureTask(final R[] resource, final Callable<T> task, - final long timeout, final int maxLockTries) { - - super(task); - - this.resource = resource; - - this.lockTimeout = timeout; - - this.maxLockTries = maxLockTries; - - this.task = task; - - } - - public LockFutureTask(final R[] resources, final Runnable task, - final T val, final long timeout, final int maxLockTries) { - - super(task, val); - - this.resource = resources; - - this.lockTimeout = timeout; - - this.maxLockTries = maxLockTries; - - this.task = task; - - } - - /** - * The resource(s) that are pre-declared by the task. {@link #call()} - * will ensure that the task as a lock on these resources before it - * invokes {@link #run()} to execution the task. - */ - public R[] getResource() { - - return resource; - - } - - /** - * The elapsed nanoseconds the task waited to acquire its locks. - */ - public long getLockLatency() { - - return nanoTime_lockLatency; - - } - - private long nanoTime_lockLatency; - - /** - * The maximum #of times that the task will attempt to acquire its locks - * (positive integer). - */ - public int getMaxLockTries() { - - return maxLockTries; - - } - - /** - * The timeout (milliseconds) or ZERO (0L) for an infinite timeout. - */ - public long getLockTimeout() { - - return lockTimeout; - - } - - /** - * Extended signal {@link NonBlockingLockManager#stateChanged} when the - * task completes, to track counters, and also exposed to the outer - * class. - */ - @Override - protected void setException(final Throwable t) { - - super.setException(t); - - lock.lock(); - try { - if (DEBUG) - log.debug("Exception: " + this + ", cause=" + t, t); - counters.nerror++; - /* - * Note: Not known to be running, hence assume waiting (this - * method is sometimes called before the task begins to - * execute). - */ - final boolean waiting = true; - releaseLocks(this,waiting); - stateChanged.signal(); - } finally { - lock.unlock(); - } - - } - - /** - * Extended signal {@link NonBlockingLockManager#stateChanged} when the - * task completes and to track counters. - */ - @Override - public boolean cancel(final boolean mayInterruptIfRunning) { - - final boolean ret = super.cancel(mayInterruptIfRunning); - - lock.lock(); - try { - if (DEBUG) - log.debug("Cancelled: " + this); - counters.ncancel++; - /* - * Note: Not known to be running, hence assume waiting (this - * method is sometimes called before the task begins to - * execute). - */ - final boolean waiting = true; - releaseLocks(this, waiting); - stateChanged.signal(); - } finally { - lock.unlock(); - } - - return ret; - - } - - /** - * Extended signal {@link NonBlockingLockManager#stateChanged} when the - * task completes and to track counters. - */ - @Override - public void run() { - - /* - * Increment by the amount of time that the task was waiting to - * acquire its lock(s). - * - * Note: This is being measured from the time when the task was - * accepted by submit() on the outer class and counts all time until - * the task begins to execute with its locks held. - */ - if (task instanceof AbstractTask - && ((AbstractTask) task).getTaskCounters() instanceof WriteTaskCounters) { - - final long lockWaitingTime = System.nanoTime() - acceptTime; - - ((WriteTaskCounters) ((AbstractTask) task).getTaskCounters()).lockWaitingNanoTime - .addAndGet(lockWaitingTime); - - } - - synchronized (counters) { - - counters.nstarted++; - - counters.nwaiting--; - - counters.nrunning++; - - if (counters.nrunning > counters.maxRunning) { - - counters.maxRunning = counters.nrunning; - - } - - } - - try { - - if(DEBUG) - log.debug("Running: "+this); - - super.run(); - - } finally { - - lock.lock(); - try { - if(DEBUG) - log.debug("Did run: "+this); - synchronized (counters) { - counters.nended++; - counters.nrunning--; - } - /* - * The task is KNOWN to not be waiting since it was running. - */ - final boolean waiting = false; - releaseLocks(this, waiting); - stateChanged.signal(); - } finally { - lock.unlock(); - } - - } - - } - - } - - /** - * Add if absent and return a {@link ResourceQueue} for the named resource. - * - * @param resource - * The resource. - * - * @return The {@link ResourceQueue}. - */ - private ResourceQueue<LockFutureTask<? extends Object>> declareResource(final R resource) { - - // test 1st to avoid creating a new ResourceQueue if it already exists. - ResourceQueue<LockFutureTask<?extends Object>> resourceQueue = resourceQueues - .get(resource); - - // not found, so create a new ResourceQueue for that resource. - resourceQueue = new ResourceQueue<LockFutureTask<?extends Object>>(resource); - - // put if absent. - final ResourceQueue<LockFutureTask<?extends Object>> oldval = resourceQueues - .putIfAbsent(resource, resourceQueue); - - if (oldval != null) { - - // concurrent insert, so use the winner's resource queue. - return oldval; - - } - - // we were the winner, so return the our new resource queue. - return resourceQueue; - - } - - /** - * Submit a task for execution. The task will wait until it holds the - * declared locks. It will then execute. This method is non-blocking. The - * caller must use {@link FutureTask#get()} to await the outcome. - * - * @param resource - * An array of resources whose locks are required to execute the - * <i>task</i>. - * @param task - * The task to be executed. - * - * @throws IllegalArgumentException - * if <i>resource</i> is <code>null</code> or if any element - * of that array is <code>null</code>. - * @throws IllegalArgumentException - * if the <i>task</i> is <code>null</code>. - * - * @throws RejectedExecutionException - * if the task can not be queued for execution (including if the - * service is not running or if a blocking queue was used and - * the queue is at capacity). - * - * @todo add variant for Runnable target. - * - * @todo get rid [lockTimeout] since you can do get(timeout) on the Future - * and the task will be cancelled unless it is complete by the - * timeout. - * - * @todo get rid of [maxLockTries] since you can specify a timeout and that - * will determine how much effort will be put into attempting to work - * around a deadlock? - */ - public <T> Future<T> submit(final R[] resource, final Callable<T> task) { - - return submit(resource, task, TimeUnit.SECONDS, - Long.MAX_VALUE/* timeout */, 1/* maxLockTries */); - - } - -// public <T> Future<T> submit(final LockCallable<R, T> task) { -// -// return submit(task.getResource(), task); -// -// } - - public <T> Future<T> submit(final R[] resource, final Callable<T> task, - final TimeUnit unit, final long lockTimeout, final int maxLockTries) { - - if (resource == null) - throw new IllegalArgumentException(); - - for (R r : resource) { - - if (r == null) - throw new IllegalArgumentException(); - - } - - if (task == null) - throw new IllegalArgumentException(); - - if (maxLockTries <= 0) - throw new IllegalArgumentException(); - - /* - * Note: We clone the resources to avoid side-effects on the caller if - * the resources are sorted (below) and also to prevent the caller from - * changing the declared locks after they submit the task. - */ - final R[] a = resource.clone(); - - if (sortLockRequests) { - - /* - * Sort the resources in the lock request. - * - * Note: Sorting the resources reduces the chance of a deadlock and - * excludes it entirely when predeclaration of locks is also used. - * - * Note: This will throw an exception if the "resource" does not - * implement Comparable. - */ - - Arrays.sort(a); - - } - - lock.lock(); - try { - - switch (runState) { - - case Running: { - - final LockFutureTask<T> future = new LockFutureTask<T>(a, task, - lockTimeout, maxLockTries); - - try { - - acceptedTasks.add(future); - - counters.naccepted++; - - } catch (IllegalStateException ex) { - - counters.nrejected++; - - throw new RejectedExecutionException(ex); - - } - - stateChanged.signal(); - - return future; - - } - - default: - - counters.nrejected++; - - throw new RejectedExecutionException("runState=" + runState); - - } - - } finally { - - lock.unlock(); - - } - - } - - /** - * Used to run the {@link AcceptTask} and the {@link MonitorTask}. - * - * FIXME Monitor this service using a {@link ThreadPoolExecutorStatisticsTask} to convert - * {@link Counters#nrunning} and {@link Counters#nwaiting} into moving - * averages. - */ - private final ExecutorService service = Executors - .newSingleThreadExecutor(new DaemonThreadFactory(getClass() - .getName())); - - /** - * The service run state. - */ - private volatile RunState runState = RunState.Starting; - - /** - * Lock used to protect changes various state changes, including change to - * the {@link #runState}. - */ - private final ReentrantLock lock = new ReentrantLock(); - - /** - * Condition is signaled whenever the {@link AcceptTask} needs to wake up. - */ - private final Condition stateChanged = lock.newCondition(); - - /** - * Tasks accepted but not yet waiting on their locks. Tasks are moved from - * here to the {@link #waitingTasks} asynchronously in order to de-couple - * the caller from {@link DeadlockException}s or periods when the system is - * at the maximum multi-programming level and can not accept another lock - * request. - */ - final private BlockingQueue<LockFutureTask<? extends Object>> acceptedTasks = new LinkedBlockingQueue<LockFutureTask<? extends Object>>(); - - /** - * Tasks whose lock requests are in the appropriate {@link ResourceQueue}s - * but which are not yet executing. - */ - private final BlockingQueue<LockFutureTask<? extends Object>> waitingTasks = new LinkedBlockingQueue<LockFutureTask<? extends Object>>(); - - /** - * {@link Runnable} drains the {@link #acceptedTasks} queue and manages - * state changes in the {@link ResourceQueue}s. Once a task is holding all - * necessary locks, the task is submitted to the delegate {@link Executor} - * for execution. This thread is also responsible for monitoring the - * {@link Future}s and releasing locks for a {@link Future} it is complete. - * - * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ - */ - private class AcceptTask implements Runnable { - - public void run() { - while (true) { - switch (runState) { - case Starting: { - awaitStateChange(Starting); - continue; - } - case Running: { - while (processAcceptedTasks() || processWaitingTasks()) { - // do work - } - awaitStateChange(Running); - continue; - } - case Shutdown: { - while (processAcceptedTasks() || processWaitingTasks()) { - /* - * Do work. - * - * Note: will run anything already accepted. That is - * intentional. Once the lock manager is shutdown it will no - * longer accept tasks, but it will process those tasks - * which it has already accepted. - */ - } - lock.lock(); - try { - if (acceptedTasks.isEmpty() && waitingTasks.isEmpty()) { - /* - * There is no more work to be performed so we can - * change the runState. - */ - if(INFO) - log.info("No more work."); - if (runState.val < RunState.ShutdownNow.val) { - setRunState(RunState.ShutdownNow); - break; - } - } - } finally { - lock.unlock(); - } - awaitStateChange(Shutdown); - continue; - } - case ShutdownNow: { - /* - * Cancel all tasks, clearing all queues. Note that only - * tasks which are on [runningTasks] need to be interrupted - * as tasks on the other queues are NOT running. - */ - if(INFO) - log.info(runState); - cancelTasks(acceptedTasks.iterator(), false/* mayInterruptIfRunning */); - cancelTasks(waitingTasks.iterator(), false/* mayInterruptIfRunning */); - lock.lock(); - try { - if (runState.val < RunState.Halted.val) { - setRunState(RunState.Halted); - } - } finally { - lock.unlock(); - } - // fall through. - } - case Halted: { - if (INFO) - log.info(runState); - // Done. - return; - } - default: - throw new AssertionError(); - } // switch(runState) - } // while(true) - } // run - - /** - * IFF there is no work that could be performed and we are in the - * expected {@link RunState} then this blocks until someone signals - * {@link NonBlockingLockManager#stateChanged}. That signal can come - * either from submitting a new task, from a running task that - * completes, from a task being cancelled, or from a task whose - * exception was set. - * - * @see LockFutureTask - * @see NonBlockingLockManager#submit(Comparable[], Callable, TimeUnit, long, int) - */ - private void awaitStateChange(final RunState expected) { - lock.lock(); - try { - /* - * While we hold the lock we verify that there really is no work - * to be done and that we are in the expected run state. Then - * and only then do we wait on [stateChanged]. - */ - if (runState != expected) { - // In a different run state. - return; - } - if (!acceptedTasks.isEmpty() || !waitingTasks.isEmpty()) { - // Some work can be done. - return; - } - if (INFO) - log.info("Waiting..."); - stateChanged.await(); - if (INFO) - log.info("Woke up..."); - } catch (InterruptedException ex) { - // someone woke us up. - } finally { - lock.unlock(); - } - } - - /** - * Cancel all tasks and remove them from the queue. - * - * @param tasks - * The tasks. - */ - private void cancelTasks( - final Iterator<LockFutureTask<? extends Object>> itr, - final boolean mayInterruptIfRunning) { - - while (itr.hasNext()) { - - final LockFutureTask<? extends Object> t = itr.next(); - - t.cancel(mayInterruptIfRunning); - - itr.remove(); - - } - - } - - /** - * Processes accepted tasks, adding lock requests for each in turn: - * <ul> - * <li> If requesting locks for a task would exceed the configured - * multi-programming level then we do not issue the request and return - * immediately. The task is left on the accepted queue and will be - * retried later.</li> - * <li>If the lock requests for a task would cause a deadlock and the - * #of retries has been exceeded, then set the {@link DeadlockException} - * on the {@link Future} and drop the task. Otherwise ntries was - * incremented and we just ignore the task for now and will retry it - * again later.</li> - * <li>If the timeout for the lock requests has already expired, then - * set set {@link TimeoutException} on the {@link Future} and drop the - * task.</li> - * </ul> - * - * @return true iff any tasks were moved to the waiting queue. - */ - private boolean processAcceptedTasks() { - - int nmoved = 0; - - final Iterator<LockFutureTask<? extends Object>> itr = acceptedTasks - .iterator(); - - while (itr.hasNext()) { - - final LockFutureTask<? extends Object> t = itr.next(); - - if (t.isCancelled()) { - - // already cancelled, e.g., by the caller. - itr.remove(); - - continue; - - } - - if (t.isTimeout()) { - - // set exception on the task. - t.setException(new java.util.concurrent.TimeoutException()); - - counters.ntimeout++; - - itr.remove(); - - continue; - - } - - int nvertices = -1; - lock.lock(); - try { - - // #of vertices before we request the locks. - if (waitsFor != null) - nvertices = waitsFor.size(); - - if (waitsFor != null && waitsFor.isFull()) { - - /* - * Note: When TxDag is used we MUST NOT add the lock - * requests if it would exceed the configured - * multi-programming capacity (an exception would be - * thrown by TxDag). Therefore we stop processing - * accepted tasks and will wait until a running task - * completes so we can start another one. - * - * Note: when tasks != transactions we need to wait - * until a transaction completes. This could require - * multiple tasks to complete if they are tasks for the - * same transaction. - */... [truncated message content] |
From: <fko...@us...> - 2010-10-08 22:22:05
|
Revision: 3761 http://bigdata.svn.sourceforge.net/bigdata/?rev=3761&view=rev Author: fkoliver Date: 2010-10-08 22:21:56 +0000 (Fri, 08 Oct 2010) Log Message: ----------- Change some subclasses of BTree into uses of BTree: EventBTree, CommitTimeIndex, CommitRecordIndex, JournalIndex, IndexSegmentIndex Users of these specialized trees should see only a specialized set of methods rather than the full inheritance from BTree. These classes should hide key/value conversion. Users should never need to see ITuple or ITupleSerializer. Removed unused CounterSetBTree. Modified Paths: -------------- branches/maven_scaleout/bigdata-core/pom.xml branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/AbstractBTree.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/BTree.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/IndexSegment.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/MetadataIndex.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/MetadataIndexView.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/Node.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/ReadOnlyIndex.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRURecycler.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/LRUNexus.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/counters/httpd/DummyEventReportingService.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/journal/AbstractJournal.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/journal/CommitRecordIndex.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/journal/DumpJournal.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/loadbalancer/EmbeddedLoadBalancer.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/resources/IndexSegmentIndex.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/resources/JournalIndex.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/resources/ResourceEvents.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/resources/StoreManager.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/AbstractTransactionService.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/CommitTimeIndex.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/DistributedTransactionService.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/EventReceiver.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/LoadBalancerService.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/MetadataIndexCache.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/transaction/EmbeddedTransactionService.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/counters/TestAll.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/journal/TestCommitRecordIndex.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/service/TestDistributedTransactionServiceRestart.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/service/TestEventReceiver.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/service/TestSnapshotHelper.java Added Paths: ----------- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/EventBTree.java Removed Paths: ------------- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/DelegateIndex.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/counters/query/CounterSetBTreeSelector.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/counters/query/CounterSetLoader.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/counters/store/CounterSetBTree.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/counters/store/TestAll.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/counters/store/TestCounterSetBTree.java Modified: branches/maven_scaleout/bigdata-core/pom.xml =================================================================== --- branches/maven_scaleout/bigdata-core/pom.xml 2010-10-08 19:57:43 UTC (rev 3760) +++ branches/maven_scaleout/bigdata-core/pom.xml 2010-10-08 22:21:56 UTC (rev 3761) @@ -60,6 +60,8 @@ java files were getting recompiled and put into the bigdata jar. This setting forces javac to only look for source in the current maven source directory. --> <sourcepath>${project.build.sourceDirectory}</sourcepath> + <!-- <Xlint></Xlint> --> + <!-- <Xlint:unchecked></Xlint:unchecked> --> </compilerArguments> </configuration> </plugin> @@ -141,7 +143,7 @@ <!-- These system properties are required by the unit tests. --> <systemPropertyVariables> <java.security.policy>${java.security.policy}</java.security.policy> - <java.net.preferIPv4Stack>{java.net.preferIPv4Stack}"</java.net.preferIPv4Stack> + <java.net.preferIPv4Stack>{java.net.preferIPv4Stack}</java.net.preferIPv4Stack> <log4j.configuration>${log4j.configuration}</log4j.configuration> <app.home>${app.home}</app.home> <!-- This is the deployment directory, easily accessed by the DataFinder class. --> @@ -160,7 +162,7 @@ <fastutil.jar>${deploy.lib}/fastutil.jar</fastutil.jar> <icu4j.jar>${deploy.lib}/icu4j.jar</icu4j.jar> <jsk-lib.jar>${deploy.lib}/jsk-lib.jar</jsk-lib.jar> - <jsk-platform.jar>${deploy.lib}jsk-platform.jar</jsk-platform.jar> + <jsk-platform.jar>${deploy.lib}/jsk-platform.jar</jsk-platform.jar> <log4j.jar>${deploy.lib}/log4j.jar</log4j.jar> <iris.jar>${deploy.lib}/iris.jar</iris.jar> <jgrapht.jar>${deploy.lib}/jgrapht.jar</jgrapht.jar> @@ -168,6 +170,13 @@ <slf4j.jar>${deploy.lib}/slf4j.jar</slf4j.jar> <nxparser.jar>${deploy.lib}/nxparser.jar</nxparser.jar> <zookeeper.jar>${deploy.lib}/zookeeper.jar</zookeeper.jar> + + + + <basedir>${deploy.dir}/testing</basedir> + + + </systemPropertyVariables> </configuration> </execution> Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/AbstractBTree.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/AbstractBTree.java 2010-10-08 19:57:43 UTC (rev 3760) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/AbstractBTree.java 2010-10-08 22:21:56 UTC (rev 3761) @@ -128,7 +128,6 @@ * </p> * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ * * @see KeyBuilder */ @@ -164,7 +163,7 @@ /** * Log for btree opeations. */ - protected static final Logger log = Logger.getLogger(AbstractBTree.class); + private static final Logger log = Logger.getLogger(AbstractBTree.class); /** * True iff the {@link #log} level is INFO or less. @@ -338,7 +337,6 @@ * * @author <a href="mailto:tho...@us...">Bryan * Thompson</a> - * @version $Id$ */ static class ChildMemoizer extends Memoizer<LoadChildRequest/* request */, AbstractNode<?>/* child */> { @@ -1362,7 +1360,6 @@ * Static class since must be {@link Serializable}. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ static final class TransientResourceMetadata implements IResourceMetadata { @@ -2588,6 +2585,7 @@ } + //fko ===== needs generic type ===== final public ITupleIterator rangeIterator() { return rangeIterator(null, null); @@ -2602,6 +2600,7 @@ * @param toKey * @return */ + //fko ===== needs generic type ===== final public ITupleIterator rangeIterator(Object fromKey, Object toKey) { fromKey = fromKey == null ? null : metadata.getTupleSerializer() @@ -2614,6 +2613,7 @@ } + //fko ===== needs generic type ===== final public ITupleIterator rangeIterator(byte[] fromKey, byte[] toKey) { return rangeIterator(fromKey, toKey, 0/* capacity */, @@ -2665,6 +2665,7 @@ * @param toKey * @return */ + //fko ===== needs generic type ===== final public ITupleIterator rangeIterator(Object fromKey, Object toKey, final int capacity,// final int flags,// @@ -2705,208 +2706,13 @@ * @todo add support to the iterator construct for filtering by a tuple * revision timestamp range. */ - public ITupleIterator rangeIterator(// - final byte[] fromKey,// - final byte[] toKey,// - final int capacityIsIgnored,// - final int flags,// - final IFilterConstructor filter// - ) { + //fko ===== needs generic type ===== + abstract public ITupleIterator rangeIterator(final byte[] fromKey, + final byte[] toKey, + final int capacityIsIgnored, + final int flags, + final IFilterConstructor filter); -// btreeCounters.nrangeIterator.incrementAndGet(); - - /* - * Does the iterator declare that it will not write back on the index? - */ - final boolean readOnly = ((flags & IRangeQuery.READONLY) != 0); - - if (readOnly && ((flags & IRangeQuery.REMOVEALL) != 0)) { - - throw new IllegalArgumentException(); - - } - - /* - * Note: this does not work out since it is not so easy to determine when - * the iterator is a point test as toKey is the exclusive upper bound. - */ -// * Note: this method will automatically apply the optional bloom filter to -// * reject range iterator requests that correspond to a point test. However -// * this can only be done when the fromKey and toKey are both non-null and -// * equals and further when the iterator was not requested with any options -// * that would permit concurrent modification of the index. -// if (isBloomFilter() -// && fromKey != null -// && toKey != null -// && (readOnly || (((flags & REMOVEALL) == 0) && ((flags & CURSOR) == 0))) -// && BytesUtil.bytesEqual(fromKey, toKey)) { -// -// /* -// * Do a fast rejection test using the bloom filter. -// */ -// if(!getBloomFilter().contains(fromKey)) { -// -// /* -// * The key is known to not be in the index so return an empty -// * iterator. -// */ -// return EmptyTupleIterator.INSTANCE; -// -// } -// -// /* -// * Since the bloom filter accepts the key we fall through into the -// * normal iterator logic. Using this code path is still possible -// * that the filter gave us a false positive and that the key is not -// * (in fact) in the index. Either way, the logic below will sort -// * things out. -// */ -// -// } - - /* - * Figure out what base iterator implementation to use. We will layer - * on the optional filter(s) below. - */ - ITupleIterator src; - - if ((this instanceof BTree) && ((flags & REVERSE) == 0) - && ((flags & REMOVEALL) == 0) && ((flags & CURSOR) == 0)) { - - /* - * Use the recursion-based striterator since it is faster for a - * BTree (but not for an IndexSegment). - * - * Note: The recursion-based striterator does not support remove()! - * - * @todo we could pass in the Tuple here to make the APIs a bit more - * consistent across the recursion-based and the cursor based - * iterators. - * - * @todo when the capacity is one and REVERSE is specified then we - * can optimize this using a reverse traversal striterator - this - * will have lower overhead than the cursor for the BTree (but not - * for an IndexSegment). - */ - -// src = fastForwardIterator(fromKey, toKey, capacity, flags); - - src = getRoot().rangeIterator(fromKey, toKey, flags); - - } else { - - final Tuple tuple = new Tuple(this, flags); - - if (this instanceof IndexSegment) { - - final IndexSegment seg = (IndexSegment) this; - - /* - * @todo we could scan the list of pools and chose the best fit - * pool and then allocate a buffer from that pool. Best fit - * would mean either the byte range fits without "too much" slop - * or the #of reads will have to perform is not too large. We - * might also want to limit the maximum size of the reads. - */ - -// final DirectBufferPool pool = DirectBufferPool.INSTANCE_10M; - final DirectBufferPool pool = DirectBufferPool.INSTANCE; - - if (true - && ((flags & REVERSE) == 0) - && ((flags & CURSOR) == 0) - && (seg.getStore().getCheckpoint().maxNodeOrLeafLength <= pool - .getBufferCapacity()) - && ((rangeCount(fromKey, toKey) / branchingFactor) > 2)) { - - src = new IndexSegmentMultiBlockIterator(seg, pool, - fromKey, toKey, flags); - - } else { - - src = new IndexSegmentTupleCursor(seg, tuple, fromKey, - toKey); - - } - - } else if (this instanceof BTree) { - - if (isReadOnly()) { - - // Note: this iterator does not allow removal. - src = new ReadOnlyBTreeTupleCursor(((BTree) this), tuple, - fromKey, toKey); - - } else { - - // Note: this iterator supports traversal with concurrent - // modification. - src = new MutableBTreeTupleCursor(((BTree) this), - new Tuple(this, flags), fromKey, toKey); - - } - - } else { - - throw new UnsupportedOperationException( - "Unknown B+Tree implementation: " - + this.getClass().getName()); - - } - - if ((flags & REVERSE) != 0) { - - /* - * Reverse scan iterator. - * - * Note: The reverse scan MUST be layered directly over the - * ITupleCursor. Most critically, REMOVEALL combined with a - * REVERSE scan needs to process the tuples in reverse index - * order and then delete them as it goes. - */ - - src = new Reverserator((ITupleCursor) src); - - } - - } - - if (filter != null) { - - /* - * Apply the optional filter. - * - * Note: This needs to be after the reverse scan and before - * REMOVEALL (those are the assumptions for the flags). - */ - - src = filter.newInstance(src); - - } - - if ((flags & REMOVEALL) != 0) { - - assertNotReadOnly(); - - /* - * Note: This iterator removes each tuple that it visits from the - * source iterator. - */ - - src = new TupleRemover() { - @Override - protected boolean remove(ITuple e) { - // remove all visited tuples. - return true; - } - }.filter(src); - - } - - return src; - - } - /** * Copy all data, including deleted index entry markers and timestamps iff * supported by the source and target. The goal is an exact copy of the data @@ -3893,7 +3699,6 @@ * {@link Reference} (a runtime security manager exception will result). * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ * * @param <T> */ Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/BTree.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/BTree.java 2010-10-08 19:57:43 UTC (rev 3760) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/BTree.java 2010-10-08 22:21:56 UTC (rev 3761) @@ -32,9 +32,13 @@ import com.bigdata.BigdataStatics; import com.bigdata.btree.AbstractBTreeTupleCursor.MutableBTreeTupleCursor; +import com.bigdata.btree.AbstractBTreeTupleCursor.ReadOnlyBTreeTupleCursor; import com.bigdata.btree.Leaf.ILeafListener; import com.bigdata.btree.data.ILeafData; import com.bigdata.btree.data.INodeData; +import com.bigdata.btree.filter.IFilterConstructor; +import com.bigdata.btree.filter.Reverserator; +import com.bigdata.btree.filter.TupleRemover; import com.bigdata.journal.AbstractJournal; import com.bigdata.journal.ICommitter; import com.bigdata.journal.IIndexManager; @@ -42,6 +46,7 @@ import com.bigdata.journal.Name2Addr.Entry; import com.bigdata.mdi.IResourceMetadata; import com.bigdata.rawstore.IRawStore; +import org.apache.log4j.Logger; /** * <p> @@ -151,10 +156,14 @@ * several published papers. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ public class BTree extends AbstractBTree implements ICommitter, ILocalBTreeView { + /** + * Log for btree operations. + */ + private static final Logger log = Logger.getLogger(BTree.class); + final public int getHeight() { return height; @@ -188,6 +197,7 @@ } + @Override public final IResourceMetadata[] getResourceMetadata() { //override to make final so sub-classes cannot modify behavior. return super.getResourceMetadata(); @@ -204,23 +214,23 @@ */ public ICounter getCounter() { - ICounter counter = new Counter(this); + ICounter tmpCounter = new Counter(this); final LocalPartitionMetadata pmd = metadata.getPartitionMetadata(); if (pmd != null) { - counter = new PartitionedCounter(pmd.getPartitionId(), counter); + tmpCounter = new PartitionedCounter(pmd.getPartitionId(), tmpCounter); } if (isReadOnly()) { - return new ReadOnlyCounter(counter); + return new ReadOnlyCounter(tmpCounter); } - return counter; + return tmpCounter; } @@ -393,7 +403,7 @@ * <p> * Note: The {@link #getCounter()} is NOT changed by this method. */ - final private void newRootLeaf() { + private void newRootLeaf() { height = 0; @@ -1339,14 +1349,8 @@ // view of this BTree. newResources[1] = new JournalMetadata((AbstractJournal) getStore(), priorCommitTime); + System.arraycopy(oldResources, 1, newResources, 2, oldResources.length - 1); - // any other stores in the view are copied. - for (int i = 1; i < oldResources.length; i++) { - - newResources[i + 1] = oldResources[i]; - - } - final LocalPartitionMetadata newPmd = new LocalPartitionMetadata( oldPmd.getPartitionId(), // partitionId -1, // sourcePartitionId @@ -1567,7 +1571,6 @@ * @throws IllegalArgumentException * if store is <code>null</code>. */ - @SuppressWarnings("unchecked") public static BTree load(final IRawStore store, final long addrCheckpoint, final boolean readOnly) { @@ -1653,6 +1656,147 @@ } /** + * Core implementation. + * <p> + * Note: If {@link IRangeQuery#CURSOR} is specified the returned iterator + * supports traversal with concurrent modification by a single-threaded + * process (the {@link BTree} is NOT thread-safe for writers). Write are + * permitted iff {@link AbstractBTree} allows writes. + * <p> + * Note: {@link IRangeQuery#REVERSE} is handled here by wrapping the + * underlying {@link ITupleCursor}. + * <p> + * Note: {@link IRangeQuery#REMOVEALL} is handled here by wrapping the + * iterator. + * <p> + * Note: + * {@link FusedView#rangeIterator(byte[], byte[], int, int, IFilterConstructor)} + * is also responsible for constructing an {@link ITupleIterator} in a + * manner similar to this method. If you are updating the logic here, then + * check the logic in that method as well! + * + * @todo add support to the iterator construct for filtering by a tuple + * revision timestamp range. + */ + public ITupleIterator rangeIterator(// + final byte[] fromKey,// + final byte[] toKey,// + final int capacityIsIgnored,// + final int flags,// + final IFilterConstructor filter// + ) { + + /* + * Does the iterator declare that it will not write back on the index? + */ + final boolean ro = ((flags & IRangeQuery.READONLY) != 0); + + if (ro && ((flags & IRangeQuery.REMOVEALL) != 0)) { + + throw new IllegalArgumentException(); + + } + + /* + * Figure out what base iterator implementation to use. We will layer + * on the optional filter(s) below. + */ + ITupleIterator src; + + if (((flags & REVERSE) == 0) && + ((flags & REMOVEALL) == 0) && + ((flags & CURSOR) == 0)) { + + /* + * Use the recursion-based striterator since it is faster for a + * BTree (but not for an IndexSegment). + * + * Note: The recursion-based striterator does not support remove()! + * + * @todo we could pass in the Tuple here to make the APIs a bit more + * consistent across the recursion-based and the cursor based + * iterators. + * + * @todo when the capacity is one and REVERSE is specified then we + * can optimize this using a reverse traversal striterator - this + * will have lower overhead than the cursor for the BTree (but not + * for an IndexSegment). + */ + src = getRoot().rangeIterator(fromKey, toKey, flags); + + } else { + + final Tuple tuple = new Tuple(this, flags); + + if (isReadOnly()) { + + // Note: this iterator does not allow removal. + src = new ReadOnlyBTreeTupleCursor(((BTree) this), tuple, + fromKey, toKey); + + } else { + + // Note: this iterator supports traversal with concurrent + // modification. + src = new MutableBTreeTupleCursor(((BTree) this), + new Tuple(this, flags), fromKey, toKey); + + } + + if ((flags & REVERSE) != 0) { + + /* + * Reverse scan iterator. + * + * Note: The reverse scan MUST be layered directly over the + * ITupleCursor. Most critically, REMOVEALL combined with a + * REVERSE scan needs to process the tuples in reverse index + * order and then delete them as it goes. + */ + + src = new Reverserator((ITupleCursor) src); + + } + + } + + if (filter != null) { + + /* + * Apply the optional filter. + * + * Note: This needs to be after the reverse scan and before + * REMOVEALL (those are the assumptions for the flags). + */ + + src = filter.newInstance(src); + + } + + if ((flags & REMOVEALL) != 0) { + + assertNotReadOnly(); + + /* + * Note: This iterator removes each tuple that it visits from the + * source iterator. + */ + + src = new TupleRemover() { + @Override + protected boolean remove(ITuple e) { + // remove all visited tuples. + return true; + } + }.filter(src); + + } + + return src; + + } + + /** * Factory for mutable nodes and leaves used by the {@link NodeSerializer}. */ protected static class NodeFactory implements INodeFactory { @@ -1682,7 +1826,6 @@ * Mutable counter. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ public static class Counter implements ICounter { @@ -1742,7 +1885,6 @@ * int32 word. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ public static class PartitionedCounter implements ICounter { @@ -1851,7 +1993,6 @@ * * @author <a href="mailto:tho...@us...">Bryan * Thompson</a> - * @version $Id$ */ protected static class Stack { @@ -2027,7 +2168,6 @@ * Note: The {@link MutableBTreeTupleCursor} does register such listeners. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ public class LeafCursor implements ILeafCursor<Leaf> { @@ -2125,6 +2265,7 @@ } + @Override public LeafCursor clone() { return new LeafCursor(this); @@ -2177,7 +2318,7 @@ } - public Leaf first() { + final public Leaf first() { stack.clear(); @@ -2198,7 +2339,7 @@ } - public Leaf last() { + final public Leaf last() { stack.clear(); @@ -2224,7 +2365,7 @@ * the leaf may not actually contain the key, in which case it is the * leaf that contains the insertion point for the key. */ - public Leaf seek(final byte[] key) { + final public Leaf seek(final byte[] key) { stack.clear(); Deleted: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/DelegateIndex.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/DelegateIndex.java 2010-10-08 19:57:43 UTC (rev 3760) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/DelegateIndex.java 2010-10-08 22:21:56 UTC (rev 3761) @@ -1,169 +0,0 @@ -/* - -Copyright (C) SYSTAP, LLC 2006-2008. All rights reserved. - -Contact: - SYSTAP, LLC - 4501 Tower Road - Greensboro, NC 27410 - lic...@bi... - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -*/ -/* - * Created on Feb 20, 2008 - */ - -package com.bigdata.btree; - -import com.bigdata.btree.filter.IFilterConstructor; -import com.bigdata.btree.proc.AbstractKeyArrayIndexProcedureConstructor; -import com.bigdata.btree.proc.IKeyRangeIndexProcedure; -import com.bigdata.btree.proc.IResultHandler; -import com.bigdata.btree.proc.ISimpleIndexProcedure; -import com.bigdata.counters.ICounterSet; -import com.bigdata.mdi.IResourceMetadata; - -/** - * An object that delegates its {@link IIndex} interface. - * - * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ - */ -public class DelegateIndex implements IIndex { - - private final IIndex delegate; - - /** - * @param delegate - * The delegate. - */ - public DelegateIndex(IIndex delegate) { - - if (delegate == null) { - - throw new IllegalArgumentException(); - - } - - this.delegate = delegate; - - } - - public String toString() { - - final StringBuilder sb = new StringBuilder(); - - sb.append(getClass().getSimpleName()); - - sb.append("{ "); - - sb.append(delegate.toString()); - - sb.append("}"); - - return sb.toString(); - - } - - public boolean contains(byte[] key) { - return delegate.contains(key); - } - - public ICounter getCounter() { - return delegate.getCounter(); - } - - public IndexMetadata getIndexMetadata() { - return delegate.getIndexMetadata(); - } - - public IResourceMetadata[] getResourceMetadata() { - return delegate.getResourceMetadata(); - } - - public ICounterSet getCounters() { - return delegate.getCounters(); - } - - public byte[] insert(byte[] key, byte[] value) { - return delegate.insert(key, value); - } - - public byte[] lookup(byte[] key) { - return delegate.lookup(key); - } - - public long rangeCount() { - return delegate.rangeCount(); - } - - public long rangeCount(byte[] fromKey, byte[] toKey) { - return delegate.rangeCount(fromKey, toKey); - } - - public long rangeCountExact(byte[] fromKey, byte[] toKey) { - return delegate.rangeCountExact(fromKey, toKey); - } - - public long rangeCountExactWithDeleted(byte[] fromKey, byte[] toKey) { - return delegate.rangeCountExactWithDeleted(fromKey, toKey); - } - - public ITupleIterator rangeIterator() { - return rangeIterator(null,null); - } - - public ITupleIterator rangeIterator(byte[] fromKey, byte[] toKey, int capacity, int flags, IFilterConstructor filter) { - return delegate.rangeIterator(fromKey, toKey, capacity, flags, filter); - } - - public ITupleIterator rangeIterator(byte[] fromKey, byte[] toKey) { - return delegate.rangeIterator(fromKey, toKey); - } - - public byte[] remove(byte[] key) { - return delegate.remove(key); - } - - public Object submit(byte[] key, ISimpleIndexProcedure proc) { - return delegate.submit(key, proc); - } - - public void submit(byte[] fromKey, byte[] toKey, IKeyRangeIndexProcedure proc, IResultHandler handler) { - delegate.submit(fromKey, toKey, proc, handler); - } - - public void submit(int fromIndex, int toIndex, byte[][] keys, byte[][] vals, AbstractKeyArrayIndexProcedureConstructor ctor, IResultHandler handler) { - delegate.submit(fromIndex, toIndex, keys, vals, ctor, handler); - } - - public boolean contains(Object key) { - return delegate.contains(key); - } - - public Object insert(Object key, Object value) { - return delegate.insert(key, value); - } - - public Object lookup(Object key) { - return delegate.lookup(key); - } - - public Object remove(Object key) { - return delegate.remove(key); - } - -} Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/IndexSegment.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/IndexSegment.java 2010-10-08 19:57:43 UTC (rev 3760) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/IndexSegment.java 2010-10-08 22:21:56 UTC (rev 3761) @@ -29,15 +29,20 @@ import com.bigdata.btree.IndexSegment.ImmutableNodeFactory.ImmutableLeaf; import com.bigdata.btree.data.ILeafData; import com.bigdata.btree.data.INodeData; +import com.bigdata.btree.filter.IFilterConstructor; +import com.bigdata.btree.filter.Reverserator; +import com.bigdata.btree.filter.TupleRemover; import com.bigdata.btree.raba.IRaba; import com.bigdata.btree.raba.ReadOnlyKeysRaba; import com.bigdata.btree.raba.ReadOnlyValuesRaba; import com.bigdata.io.AbstractFixedByteArrayBuffer; +import com.bigdata.io.DirectBufferPool; import com.bigdata.io.FixedByteArrayBuffer; import com.bigdata.mdi.IResourceMetadata; import com.bigdata.service.Event; import com.bigdata.service.EventResource; import com.bigdata.service.EventType; +import org.apache.log4j.Logger; /** * An index segment is read-only btree corresponding to some key range of a @@ -51,11 +56,15 @@ * leaves will all refuse mutation operations). * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ public class IndexSegment extends AbstractBTree { /** + * Log for btree opeations. + */ + private static final Logger log = Logger.getLogger(IndexSegment.class); + + /** * Type safe reference to the backing store. */ private final IndexSegmentStore fileStore; @@ -160,6 +169,7 @@ * {@link #getEntryCount()} uses the {@link IndexSegmentCheckpoint} and that * is only available while the {@link IndexSegmentStore} is open. */ + @Override public String toString() { // make sure the fileStore will remain open. @@ -303,7 +313,7 @@ } @Override - protected void _reopen() { + final protected void _reopen() { // prevent concurrent close. fileStore.lock.lock(); @@ -684,6 +694,138 @@ }; } + /** + * Core implementation. + * <p> + * Note: If {@link IRangeQuery#CURSOR} is specified the returned iterator + * supports traversal with concurrent modification by a single-threaded + * process (the {@link BTree} is NOT thread-safe for writers). Write are + * permitted iff {@link AbstractBTree} allows writes. + * <p> + * Note: {@link IRangeQuery#REVERSE} is handled here by wrapping the + * underlying {@link ITupleCursor}. + * <p> + * Note: {@link IRangeQuery#REMOVEALL} is handled here by wrapping the + * iterator. + * <p> + * Note: + * {@link FusedView#rangeIterator(byte[], byte[], int, int, IFilterConstructor)} + * is also responsible for constructing an {@link ITupleIterator} in a + * manner similar to this method. If you are updating the logic here, then + * check the logic in that method as well! + * + * @todo add support to the iterator construct for filtering by a tuple + * revision timestamp range. + */ + public ITupleIterator rangeIterator(// + final byte[] fromKey,// + final byte[] toKey,// + final int capacityIsIgnored,// + final int flags,// + final IFilterConstructor filter// + ) { + + /* + * Does the iterator declare that it will not write back on the index? + */ + final boolean readOnly = ((flags & IRangeQuery.READONLY) != 0); + + if (readOnly && ((flags & IRangeQuery.REMOVEALL) != 0)) { + + throw new IllegalArgumentException(); + + } + + /* + * Figure out what base iterator implementation to use. We will layer + * on the optional filter(s) below. + */ + ITupleIterator src; + + final Tuple tuple = new Tuple(this, flags); + + final IndexSegment seg = (IndexSegment) this; + + /* + * @todo we could scan the list of pools and chose the best fit + * pool and then allocate a buffer from that pool. Best fit + * would mean either the byte range fits without "too much" slop + * or the #of reads will have to perform is not too large. We + * might also want to limit the maximum size of the reads. + */ + +// final DirectBufferPool pool = DirectBufferPool.INSTANCE_10M; + final DirectBufferPool pool = DirectBufferPool.INSTANCE; + + if (true + && ((flags & REVERSE) == 0) + && ((flags & CURSOR) == 0) + && (seg.getStore().getCheckpoint().maxNodeOrLeafLength <= pool + .getBufferCapacity()) + && ((rangeCount(fromKey, toKey) / branchingFactor) > 2)) { + + src = new IndexSegmentMultiBlockIterator(seg, pool, + fromKey, toKey, flags); + + } else { + + src = new IndexSegmentTupleCursor(seg, tuple, fromKey, toKey); + + } + + + if ((flags & REVERSE) != 0) { + + /* + * Reverse scan iterator. + * + * Note: The reverse scan MUST be layered directly over the + * ITupleCursor. Most critically, REMOVEALL combined with a + * REVERSE scan needs to process the tuples in reverse index + * order and then delete them as it goes. + */ + + src = new Reverserator((ITupleCursor) src); + + } + + + if (filter != null) { + + /* + * Apply the optional filter. + * + * Note: This needs to be after the reverse scan and before + * REMOVEALL (those are the assumptions for the flags). + */ + + src = filter.newInstance(src); + + } + + if ((flags & REMOVEALL) != 0) { + + assertNotReadOnly(); + + /* + * Note: This iterator removes each tuple that it visits from the + * source iterator. + */ + + src = new TupleRemover() { + @Override + protected boolean remove(ITuple e) { + // remove all visited tuples. + return true; + } + }.filter(src); + + } + + return src; + + } + /* * INodeFactory */ @@ -719,7 +861,6 @@ * * @author <a href="mailto:tho...@us...">Bryan * Thompson</a> - * @version $Id$ */ public static class ImmutableNode extends Node { @@ -768,8 +909,6 @@ * * @author <a href="mailto:tho...@us...">Bryan * Thompson</a> - * @version $Id: IndexSegment.java 2265 2009-10-26 12:51:06Z thompsonbry - * $ */ private static class EmptyReadOnlyLeafData implements ILeafData { @@ -869,7 +1008,6 @@ * * @author <a href="mailto:tho...@us...">Bryan * Thompson</a> - * @version $Id$ */ public static class ImmutableLeaf extends Leaf { @@ -1007,7 +1145,6 @@ // * // * @author <a href="mailto:tho...@us...">Bryan // * Thompson</a> -// * @version $Id$ // */ // static public class ImmutableEmptyLastLeaf extends ImmutableLeaf { // @@ -1046,7 +1183,6 @@ * A position for the {@link IndexSegmentTupleCursor}. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ * @param <E> * The generic type for objects de-serialized from the values in * the index. @@ -1054,6 +1190,7 @@ static private class CursorPosition<E> extends AbstractCursorPosition<ImmutableLeaf,E> { @SuppressWarnings("unchecked") + @Override public IndexSegmentTupleCursor<E> getCursor() { return (IndexSegmentTupleCursor)cursor; @@ -1102,7 +1239,6 @@ * listeners for concurrent modifications. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ * @param <E> * The generic type for the objects de-serialized from the index. */ @@ -1265,7 +1401,6 @@ * Cursor using the double-linked leaves for efficient scans. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ public class ImmutableLeafCursor implements ILeafCursor<ImmutableLeaf> { @@ -1283,6 +1418,7 @@ } + @Override public ImmutableLeafCursor clone() { return new ImmutableLeafCursor(this); @@ -1333,7 +1469,7 @@ } - public ImmutableLeaf seek(final byte[] key) { + final public ImmutableLeaf seek(final byte[] key) { leaf = findLeaf(key); @@ -1363,7 +1499,7 @@ } - public ImmutableLeaf first() { + final public ImmutableLeaf first() { final long addr = getStore().getCheckpoint().addrFirstLeaf; @@ -1373,7 +1509,7 @@ } - public ImmutableLeaf last() { + final public ImmutableLeaf last() { final long addr = getStore().getCheckpoint().addrLastLeaf; Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/MetadataIndex.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/MetadataIndex.java 2010-10-08 19:57:43 UTC (rev 3760) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/MetadataIndex.java 2010-10-08 22:21:56 UTC (rev 3761) @@ -29,13 +29,13 @@ import java.io.ObjectOutput; import java.util.UUID; -import com.bigdata.mdi.PartitionLocator; import org.CognitiveWeb.extser.LongPacker; import com.bigdata.btree.keys.IKeyBuilderFactory; import com.bigdata.btree.view.FusedView; import com.bigdata.journal.ICommitter; import com.bigdata.journal.IResourceManager; +import com.bigdata.mdi.PartitionLocator; import com.bigdata.rawstore.IRawStore; import com.bigdata.service.MetadataService; @@ -44,7 +44,7 @@ * metadata index for each distributed index. The keys of the metadata index are * the first key that would be directed into the corresponding index segment, * e.g., a <em>separator key</em> (this is just the standard btree semantics). - * The values are serialized {@link com.bigdata.mdi.PartitionLocator} objects. + * The values are serialized {@link PartitionLocator} objects. * <p> * Note: At this time the recommended scale-out approach for the metadata index * is to place the metadata indices on a {@link MetadataService} (the same @@ -63,7 +63,6 @@ * taking the database offline. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ * * @todo The {@link MetadataIndex} does NOT support either overflow (it may NOT * be a {@link FusedView}) NOR key-range splits. There are several issues @@ -97,6 +96,7 @@ */ private transient final MetadataIndexView view; + @Override public MetadataIndexMetadata getIndexMetadata() { return (MetadataIndexMetadata) super.getIndexMetadata(); @@ -210,6 +210,7 @@ * Extended to require a checkpoint if {@link #incrementAndGetNextPartitionId()} has been * invoked. */ + @Override public boolean needsCheckpoint() { if(nextPartitionId != ((MetadataIndexCheckpoint)getCheckpoint()).getNextPartitionId()) { @@ -227,7 +228,6 @@ * identifier to be assigned by the metadata index. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ public static class MetadataIndexCheckpoint extends Checkpoint { @@ -329,7 +329,6 @@ * for the managed scale-out index. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ public static class MetadataIndexMetadata extends IndexMetadata implements Externalizable { @@ -381,6 +380,7 @@ private static final transient int VERSION0 = 0x0; + @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { @@ -398,6 +398,7 @@ } + @Override public void writeExternal(ObjectOutput out) throws IOException { super.writeExternal(out); @@ -430,7 +431,6 @@ * {@link MetadataIndex}. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ public static class PartitionLocatorTupleSerializer extends DefaultTupleSerializer<byte[]/*key*/, PartitionLocator/*val*/> { @@ -473,6 +473,7 @@ */ private final static transient byte VERSION = VERSION0; + @Override public void readExternal(final ObjectInput in) throws IOException, ClassNotFoundException { @@ -490,6 +491,7 @@ } + @Override public void writeExternal(final ObjectOutput out) throws IOException { super.writeExternal(out); Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/MetadataIndexView.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/MetadataIndexView.java 2010-10-08 19:57:43 UTC (rev 3760) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/MetadataIndexView.java 2010-10-08 22:21:56 UTC (rev 3761) @@ -28,27 +28,27 @@ package com.bigdata.btree; -import com.bigdata.mdi.PartitionLocator; +import com.bigdata.btree.MetadataIndex.MetadataIndexMetadata; +import com.bigdata.btree.filter.IFilterConstructor; import org.apache.log4j.Logger; import com.bigdata.cache.LRUCache; -import com.bigdata.btree.MetadataIndex.MetadataIndexMetadata; +import com.bigdata.mdi.PartitionLocator; /** * The extension semantics for the {@link IMetadataIndex} are implemented by * this class. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ -public class MetadataIndexView extends DelegateIndex implements IMetadataIndex { +public class MetadataIndexView implements IMetadataIndex { protected static final Logger log = Logger.getLogger(MetadataIndexView.class); // protected static final boolean INFO = log.isInfoEnabled(); // protected static final boolean DEBUG = log.isDebugEnabled(); - private final AbstractBTree delegate; + private final MetadataIndex metadataIndex; /** * <code>true</code> iff this is a read-only view. this is used to @@ -58,19 +58,14 @@ */ final private boolean readOnly; - public MetadataIndexView(AbstractBTree delegate) { - - super(delegate); - - this.delegate = delegate; - - this.readOnly = delegate.isReadOnly(); - + public MetadataIndexView(MetadataIndex metadataIndex) { + this.metadataIndex = metadataIndex; + this.readOnly = metadataIndex.isReadOnly(); } public MetadataIndexMetadata getIndexMetadata() { - return (MetadataIndexMetadata) super.getIndexMetadata(); + return (MetadataIndexMetadata) metadataIndex.getIndexMetadata(); } @@ -87,7 +82,7 @@ * de-serialization using the ITupleSerializer. */ - return (PartitionLocator) delegate.lookup((Object) key); + return (PartitionLocator) metadataIndex.lookup((Object) key); } @@ -112,7 +107,7 @@ */ private PartitionLocator find_with_iterator(byte[] key) { - final ITupleIterator<PartitionLocator> itr = delegate.rangeIterator( + final ITupleIterator<PartitionLocator> itr = metadataIndex.rangeIterator( null/* fromKey */, key/* toKey */, 1/* capacity */, IRangeQuery.VALS | IRangeQuery.REVERSE, null/* filter */); @@ -147,7 +142,7 @@ if (key == null) { // use the index of the last partition. - index = delegate.getEntryCount() - 1; + index = metadataIndex.getEntryCount() - 1; } else { @@ -212,7 +207,7 @@ /** * Remove the locator from the {@link #locatorCache}. It will be re-read on - * demand from the {@link #delegate}. + * demand from the {@link #metadataIndex}. */ public void staleLocator(PartitionLocator locator) { @@ -230,8 +225,8 @@ */ private PartitionLocator getLocatorAtIndex(int index) { - final ITuple<PartitionLocator> tuple = delegate.valueAt(index, - delegate.getLookupTuple()); + final ITuple<PartitionLocator> tuple = metadataIndex.valueAt(index, + metadataIndex.getLookupTuple()); return tuple.getObject(); @@ -252,7 +247,7 @@ */ private int findIndexOf(byte[] key) { - int pos = delegate.indexOf(key); + int pos = metadataIndex.indexOf(key); if (pos < 0) { @@ -266,7 +261,7 @@ if(pos == 0) { - if(delegate.getEntryCount() != 0) { + if(metadataIndex.getEntryCount() != 0) { throw new IllegalStateException( "Partition not defined for empty key."); @@ -294,4 +289,34 @@ } + public long rangeCount() { + return metadataIndex.rangeCount(); + } + + public long rangeCount(byte[] fromKey, byte[] toKey) { + return metadataIndex.rangeCount(fromKey, toKey); + } + + public long rangeCountExact(byte[] fromKey, byte[] toKey) { + return metadataIndex.rangeCountExact(fromKey, toKey); + } + + public long rangeCountExactWithDeleted(byte[] fromKey, byte[] toKey) { + return metadataIndex.rangeCountExactWithDeleted(fromKey, toKey); + } + + public ITupleIterator rangeIterator() { + return metadataIndex.rangeIterator(); + } + + public ITupleIterator rangeIterator(byte[] fromKey, byte[] toKey) { + return metadataIndex.rangeIterator(fromKey, toKey); + } + + public ITupleIterator rangeIterator(byte[] fromKey, byte[] toKey, + int capacity, int flags, + IFilterConstructor filterCtor) { + return metadataIndex.rangeIterator(fromKey, toKey, capacity, flags, filterCtor); + } + } Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/Node.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/Node.java 2010-10-08 19:57:43 UTC (rev 3760) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/Node.java 2010-10-08 22:21:56 UTC (rev 3761) @@ -56,6 +56,7 @@ import cutthecrap.utils.striterators.Expander; import cutthecrap.utils.striterators.SingleValueIterator; import cutthecrap.utils.striterators.Striterator; +import org.apache.log4j.Logger; /** * <p> @@ -82,11 +83,15 @@ * we can prune the search before we materialize the child. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ public class Node extends AbstractNode<Node> implements INodeData { /** + * Log for btree opeations. + */ + private static final Logger log = Logger.getLogger(Node.class); + + /** * The data record. {@link MutableNodeData} is used for all mutation * operations. {@link ReadOnlyNodeData} is used when the {@link Node} is * made persistent. A read-only data record is automatically converted into @@ -623,7 +628,7 @@ btree.getBtreeCounters().rootsSplit++; - if (BTree.log.isInfoEnabled() || BigdataStatics.debug) { + if (log.isInfoEnabled() || BigdataStatics.debug) { // Note: nnodes and nleaves might not reflect rightSibling yet. @@ -632,8 +637,8 @@ + ", m=" + btree.getBranchingFactor() + ", nentries=" + btree.nentries; - if (BTree.log.isInfoEnabled()) - BTree.log.info(msg); + if (log.isInfoEnabled()) + log.info(msg); if (BigdataStatics.debug) System.err.println(msg); @@ -2400,8 +2405,8 @@ // one less node in the tree. btree.nnodes--; - if (BTree.INFO) { - BTree.log.info("reduced tree height: height=" + if (INFO) { + log.info("reduced tree height: height=" + btree.height + ", newRoot=" + btree.root); } Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/ReadOnlyIndex.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/ReadOnlyIndex.java 2010-10-08 19:57:43 UTC (rev 3760) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/ReadOnlyIndex.java 2010-10-08 22:21:56 UTC (rev 3761) @@ -27,6 +27,9 @@ package com.bigdata.btree; +import com.bigdata.btree.proc.IKeyRangeIndexProcedure; +import com.bigdata.btree.proc.ISimpleIndexProcedure; +import com.bigdata.counters.ICounterSet; import java.util.Iterator; import com.bigdata.btree.filter.IFilterConstructor; @@ -47,20 +50,19 @@ * @see {@link IResourceManager#getIndex(String, long)} * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ -public class ReadOnlyIndex extends DelegateIndex { +public class ReadOnlyIndex implements IIndex { + + private IIndex src; public ReadOnlyIndex(IIndex src) { - - super(src); - + this.src = src; } /** {@link IndexMetadata} is cloned to disallow modification. */ final public IndexMetadata getIndexMetadata() { - return super.getIndexMetadata().clone(); + return src.getIndexMetadata().clone(); } @@ -71,7 +73,7 @@ */ final public IResourceMetadata[] getResourceMetadata() { - return super.getResourceMetadata().clone(); + return src.getResourceMetadata().clone(); } @@ -80,7 +82,7 @@ */ final public ICounter getCounter() { - return new ReadOnlyCounter(super.getCounter()); + return new ReadOnlyCounter(src.getCounter()); } @@ -121,7 +123,7 @@ /* * Must explicitly disable Iterator#remove(). */ - return new ReadOnlyEntryIterator(super.rangeIterator(fromKey, toKey, + return new ReadOnlyEntryIterator(src.rangeIterator(fromKey, toKey, capacity, flags, filter)); } @@ -173,4 +175,65 @@ } + public ICounterSet getCounters() { + return src.getCounters(); + } + + public Object submit(byte[] key, ISimpleIndexProcedure proc) { + return src.submit(key, proc); + } + + public void submit(byte[] fromKey, byte[] toKey, + IKeyRangeIndexProcedure proc, IResultHandler handler) { + src.submit(fromKey, toKey, proc, handler); + } + + public byte[] lookup(byte[] key) { + return src.lookup(key); + } + + public boolean contains(byte[] key) { + return src.contains(key); + } + + public Object insert(Object key, Object value) { + return src.insert(key, value); + } + + public Object lookup(Object key) { + return src.lookup(key); + } + + public boolean contains(Object key) { + return src.contains(key); + } + + public Object remove(Object key) { + return src.remove(key); + } + + public long rangeCount() { + return src.rangeCount(); + } + + public long rangeCount(byte[] fromKey, byte[] toKey) { + return src.rangeCount(fromKey, toKey); + } + + public long rangeCountExact(byte[] fromKey, byte[] toKey) { + return src.rangeCountExact(fromKey, toKey); + } + + public long rangeCountExactWithDeleted(byte[] fromKey, byte[] toKey) { + return src.rangeCountExactWithDeleted(fromKey, toKey); + } + + public ITupleIterator rangeIterator() { + return src.rangeIterator(); + } + + public ITupleIterator rangeIterator(byte[] fromKey, byte[] toKey) { + return src.rangeIterator(fromKey, toKey); + } + } Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRURecycler.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRURecycler.java 2010-10-08 19:57:43 UTC (rev 3760) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRURecycler.java 2010-10-08 22:21:56 UTC (rev 3761) @@ -613,6 +613,7 @@ e.prior = last; last = e; } + //System.err.println("++size"); size++; counters.bytesInMemory.addAndGet(e.bytesInMemory); counters.bytesOnDisk.addAndGet(e.bytesOnDisk); @@ -654,6 +655,7 @@ e.cache = null; // clear reference to the cache. e.k = null; // clear the key. e.v = null; // clear the value reference. + //System.err.println("--size"); size--; counters.bytesInMemory.addAndGet(-e.bytesInMemory); counters.bytesOnDisk.addAndGet(-e.bytesOnDisk); Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/LRUNexus.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/LRUNexus.java 2010-10-08 19:57:43 UTC (rev 3760) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/LRUNexus.java 2010-10-08 22:21:56 UTC (rev 3761) @@ -47,7 +47,6 @@ * system properties defined by {@link Options}. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ * * FIXME LR... [truncated message content] |
From: <fko...@us...> - 2010-10-14 20:44:56
|
Revision: 3801 http://bigdata.svn.sourceforge.net/bigdata/?rev=3801&view=rev Author: fkoliver Date: 2010-10-14 20:44:47 +0000 (Thu, 14 Oct 2010) Log Message: ----------- Revert r3761 which changed some subclasses of BTree into uses of BTree. Modified Paths: -------------- branches/maven_scaleout/bigdata-core/pom.xml branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/AbstractBTree.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/BTree.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/IndexSegment.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/MetadataIndex.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/MetadataIndexView.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/Node.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/ReadOnlyIndex.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRURecycler.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/LRUNexus.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/counters/httpd/DummyEventReportingService.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/journal/AbstractJournal.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/journal/CommitRecordIndex.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/journal/DumpJournal.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/loadbalancer/EmbeddedLoadBalancer.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/resources/IndexSegmentIndex.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/resources/JournalIndex.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/resources/ResourceEvents.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/resources/StoreManager.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/AbstractTransactionService.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/CommitTimeIndex.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/DistributedTransactionService.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/EventReceiver.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/LoadBalancerService.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/MetadataIndexCache.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/transaction/EmbeddedTransactionService.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/counters/TestAll.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/journal/TestCommitRecordIndex.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/service/TestDistributedTransactionServiceRestart.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/service/TestEventReceiver.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/service/TestSnapshotHelper.java Added Paths: ----------- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/DelegateIndex.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/counters/query/CounterSetBTreeSelector.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/counters/query/CounterSetLoader.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/counters/store/CounterSetBTree.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/counters/store/TestAll.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/counters/store/TestCounterSetBTree.java Removed Paths: ------------- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/EventBTree.java Modified: branches/maven_scaleout/bigdata-core/pom.xml =================================================================== --- branches/maven_scaleout/bigdata-core/pom.xml 2010-10-14 16:19:22 UTC (rev 3800) +++ branches/maven_scaleout/bigdata-core/pom.xml 2010-10-14 20:44:47 UTC (rev 3801) @@ -60,8 +60,6 @@ java files were getting recompiled and put into the bigdata jar. This setting forces javac to only look for source in the current maven source directory. --> <sourcepath>${project.build.sourceDirectory}</sourcepath> - <!-- <Xlint></Xlint> --> - <!-- <Xlint:unchecked></Xlint:unchecked> --> </compilerArguments> </configuration> </plugin> @@ -162,7 +160,7 @@ <fastutil.jar>${deploy.lib}/fastutil.jar</fastutil.jar> <icu4j.jar>${deploy.lib}/icu4j.jar</icu4j.jar> <jsk-lib.jar>${deploy.lib}/jsk-lib.jar</jsk-lib.jar> - <jsk-platform.jar>${deploy.lib}/jsk-platform.jar</jsk-platform.jar> + <jsk-platform.jar>${deploy.lib}jsk-platform.jar</jsk-platform.jar> <log4j.jar>${deploy.lib}/log4j.jar</log4j.jar> <iris.jar>${deploy.lib}/iris.jar</iris.jar> <jgrapht.jar>${deploy.lib}/jgrapht.jar</jgrapht.jar> Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/AbstractBTree.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/AbstractBTree.java 2010-10-14 16:19:22 UTC (rev 3800) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/AbstractBTree.java 2010-10-14 20:44:47 UTC (rev 3801) @@ -128,6 +128,7 @@ * </p> * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> + * @version $Id$ * * @see KeyBuilder */ @@ -163,7 +164,7 @@ /** * Log for btree opeations. */ - private static final Logger log = Logger.getLogger(AbstractBTree.class); + protected static final Logger log = Logger.getLogger(AbstractBTree.class); /** * True iff the {@link #log} level is INFO or less. @@ -337,6 +338,7 @@ * * @author <a href="mailto:tho...@us...">Bryan * Thompson</a> + * @version $Id$ */ static class ChildMemoizer extends Memoizer<LoadChildRequest/* request */, AbstractNode<?>/* child */> { @@ -1360,6 +1362,7 @@ * Static class since must be {@link Serializable}. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> + * @version $Id$ */ static final class TransientResourceMetadata implements IResourceMetadata { @@ -2585,7 +2588,6 @@ } - //fko ===== needs generic type ===== final public ITupleIterator rangeIterator() { return rangeIterator(null, null); @@ -2600,7 +2602,6 @@ * @param toKey * @return */ - //fko ===== needs generic type ===== final public ITupleIterator rangeIterator(Object fromKey, Object toKey) { fromKey = fromKey == null ? null : metadata.getTupleSerializer() @@ -2613,7 +2614,6 @@ } - //fko ===== needs generic type ===== final public ITupleIterator rangeIterator(byte[] fromKey, byte[] toKey) { return rangeIterator(fromKey, toKey, 0/* capacity */, @@ -2665,7 +2665,6 @@ * @param toKey * @return */ - //fko ===== needs generic type ===== final public ITupleIterator rangeIterator(Object fromKey, Object toKey, final int capacity,// final int flags,// @@ -2706,13 +2705,208 @@ * @todo add support to the iterator construct for filtering by a tuple * revision timestamp range. */ - //fko ===== needs generic type ===== - abstract public ITupleIterator rangeIterator(final byte[] fromKey, - final byte[] toKey, - final int capacityIsIgnored, - final int flags, - final IFilterConstructor filter); + public ITupleIterator rangeIterator(// + final byte[] fromKey,// + final byte[] toKey,// + final int capacityIsIgnored,// + final int flags,// + final IFilterConstructor filter// + ) { +// btreeCounters.nrangeIterator.incrementAndGet(); + + /* + * Does the iterator declare that it will not write back on the index? + */ + final boolean readOnly = ((flags & IRangeQuery.READONLY) != 0); + + if (readOnly && ((flags & IRangeQuery.REMOVEALL) != 0)) { + + throw new IllegalArgumentException(); + + } + + /* + * Note: this does not work out since it is not so easy to determine when + * the iterator is a point test as toKey is the exclusive upper bound. + */ +// * Note: this method will automatically apply the optional bloom filter to +// * reject range iterator requests that correspond to a point test. However +// * this can only be done when the fromKey and toKey are both non-null and +// * equals and further when the iterator was not requested with any options +// * that would permit concurrent modification of the index. +// if (isBloomFilter() +// && fromKey != null +// && toKey != null +// && (readOnly || (((flags & REMOVEALL) == 0) && ((flags & CURSOR) == 0))) +// && BytesUtil.bytesEqual(fromKey, toKey)) { +// +// /* +// * Do a fast rejection test using the bloom filter. +// */ +// if(!getBloomFilter().contains(fromKey)) { +// +// /* +// * The key is known to not be in the index so return an empty +// * iterator. +// */ +// return EmptyTupleIterator.INSTANCE; +// +// } +// +// /* +// * Since the bloom filter accepts the key we fall through into the +// * normal iterator logic. Using this code path is still possible +// * that the filter gave us a false positive and that the key is not +// * (in fact) in the index. Either way, the logic below will sort +// * things out. +// */ +// +// } + + /* + * Figure out what base iterator implementation to use. We will layer + * on the optional filter(s) below. + */ + ITupleIterator src; + + if ((this instanceof BTree) && ((flags & REVERSE) == 0) + && ((flags & REMOVEALL) == 0) && ((flags & CURSOR) == 0)) { + + /* + * Use the recursion-based striterator since it is faster for a + * BTree (but not for an IndexSegment). + * + * Note: The recursion-based striterator does not support remove()! + * + * @todo we could pass in the Tuple here to make the APIs a bit more + * consistent across the recursion-based and the cursor based + * iterators. + * + * @todo when the capacity is one and REVERSE is specified then we + * can optimize this using a reverse traversal striterator - this + * will have lower overhead than the cursor for the BTree (but not + * for an IndexSegment). + */ + +// src = fastForwardIterator(fromKey, toKey, capacity, flags); + + src = getRoot().rangeIterator(fromKey, toKey, flags); + + } else { + + final Tuple tuple = new Tuple(this, flags); + + if (this instanceof IndexSegment) { + + final IndexSegment seg = (IndexSegment) this; + + /* + * @todo we could scan the list of pools and chose the best fit + * pool and then allocate a buffer from that pool. Best fit + * would mean either the byte range fits without "too much" slop + * or the #of reads will have to perform is not too large. We + * might also want to limit the maximum size of the reads. + */ + +// final DirectBufferPool pool = DirectBufferPool.INSTANCE_10M; + final DirectBufferPool pool = DirectBufferPool.INSTANCE; + + if (true + && ((flags & REVERSE) == 0) + && ((flags & CURSOR) == 0) + && (seg.getStore().getCheckpoint().maxNodeOrLeafLength <= pool + .getBufferCapacity()) + && ((rangeCount(fromKey, toKey) / branchingFactor) > 2)) { + + src = new IndexSegmentMultiBlockIterator(seg, pool, + fromKey, toKey, flags); + + } else { + + src = new IndexSegmentTupleCursor(seg, tuple, fromKey, + toKey); + + } + + } else if (this instanceof BTree) { + + if (isReadOnly()) { + + // Note: this iterator does not allow removal. + src = new ReadOnlyBTreeTupleCursor(((BTree) this), tuple, + fromKey, toKey); + + } else { + + // Note: this iterator supports traversal with concurrent + // modification. + src = new MutableBTreeTupleCursor(((BTree) this), + new Tuple(this, flags), fromKey, toKey); + + } + + } else { + + throw new UnsupportedOperationException( + "Unknown B+Tree implementation: " + + this.getClass().getName()); + + } + + if ((flags & REVERSE) != 0) { + + /* + * Reverse scan iterator. + * + * Note: The reverse scan MUST be layered directly over the + * ITupleCursor. Most critically, REMOVEALL combined with a + * REVERSE scan needs to process the tuples in reverse index + * order and then delete them as it goes. + */ + + src = new Reverserator((ITupleCursor) src); + + } + + } + + if (filter != null) { + + /* + * Apply the optional filter. + * + * Note: This needs to be after the reverse scan and before + * REMOVEALL (those are the assumptions for the flags). + */ + + src = filter.newInstance(src); + + } + + if ((flags & REMOVEALL) != 0) { + + assertNotReadOnly(); + + /* + * Note: This iterator removes each tuple that it visits from the + * source iterator. + */ + + src = new TupleRemover() { + @Override + protected boolean remove(ITuple e) { + // remove all visited tuples. + return true; + } + }.filter(src); + + } + + return src; + + } + /** * Copy all data, including deleted index entry markers and timestamps iff * supported by the source and target. The goal is an exact copy of the data @@ -3699,6 +3893,7 @@ * {@link Reference} (a runtime security manager exception will result). * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> + * @version $Id$ * * @param <T> */ Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/BTree.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/BTree.java 2010-10-14 16:19:22 UTC (rev 3800) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/BTree.java 2010-10-14 20:44:47 UTC (rev 3801) @@ -32,13 +32,9 @@ import com.bigdata.BigdataStatics; import com.bigdata.btree.AbstractBTreeTupleCursor.MutableBTreeTupleCursor; -import com.bigdata.btree.AbstractBTreeTupleCursor.ReadOnlyBTreeTupleCursor; import com.bigdata.btree.Leaf.ILeafListener; import com.bigdata.btree.data.ILeafData; import com.bigdata.btree.data.INodeData; -import com.bigdata.btree.filter.IFilterConstructor; -import com.bigdata.btree.filter.Reverserator; -import com.bigdata.btree.filter.TupleRemover; import com.bigdata.journal.AbstractJournal; import com.bigdata.journal.ICommitter; import com.bigdata.journal.IIndexManager; @@ -46,7 +42,6 @@ import com.bigdata.journal.Name2Addr.Entry; import com.bigdata.mdi.IResourceMetadata; import com.bigdata.rawstore.IRawStore; -import org.apache.log4j.Logger; /** * <p> @@ -156,14 +151,10 @@ * several published papers. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> + * @version $Id$ */ public class BTree extends AbstractBTree implements ICommitter, ILocalBTreeView { - /** - * Log for btree operations. - */ - private static final Logger log = Logger.getLogger(BTree.class); - final public int getHeight() { return height; @@ -197,7 +188,6 @@ } - @Override public final IResourceMetadata[] getResourceMetadata() { //override to make final so sub-classes cannot modify behavior. return super.getResourceMetadata(); @@ -214,23 +204,23 @@ */ public ICounter getCounter() { - ICounter tmpCounter = new Counter(this); + ICounter counter = new Counter(this); final LocalPartitionMetadata pmd = metadata.getPartitionMetadata(); if (pmd != null) { - tmpCounter = new PartitionedCounter(pmd.getPartitionId(), tmpCounter); + counter = new PartitionedCounter(pmd.getPartitionId(), counter); } if (isReadOnly()) { - return new ReadOnlyCounter(tmpCounter); + return new ReadOnlyCounter(counter); } - return tmpCounter; + return counter; } @@ -403,7 +393,7 @@ * <p> * Note: The {@link #getCounter()} is NOT changed by this method. */ - private void newRootLeaf() { + final private void newRootLeaf() { height = 0; @@ -1349,8 +1339,14 @@ // view of this BTree. newResources[1] = new JournalMetadata((AbstractJournal) getStore(), priorCommitTime); - System.arraycopy(oldResources, 1, newResources, 2, oldResources.length - 1); + // any other stores in the view are copied. + for (int i = 1; i < oldResources.length; i++) { + + newResources[i + 1] = oldResources[i]; + + } + final LocalPartitionMetadata newPmd = new LocalPartitionMetadata( oldPmd.getPartitionId(), // partitionId -1, // sourcePartitionId @@ -1571,6 +1567,7 @@ * @throws IllegalArgumentException * if store is <code>null</code>. */ + @SuppressWarnings("unchecked") public static BTree load(final IRawStore store, final long addrCheckpoint, final boolean readOnly) { @@ -1656,147 +1653,6 @@ } /** - * Core implementation. - * <p> - * Note: If {@link IRangeQuery#CURSOR} is specified the returned iterator - * supports traversal with concurrent modification by a single-threaded - * process (the {@link BTree} is NOT thread-safe for writers). Write are - * permitted iff {@link AbstractBTree} allows writes. - * <p> - * Note: {@link IRangeQuery#REVERSE} is handled here by wrapping the - * underlying {@link ITupleCursor}. - * <p> - * Note: {@link IRangeQuery#REMOVEALL} is handled here by wrapping the - * iterator. - * <p> - * Note: - * {@link FusedView#rangeIterator(byte[], byte[], int, int, IFilterConstructor)} - * is also responsible for constructing an {@link ITupleIterator} in a - * manner similar to this method. If you are updating the logic here, then - * check the logic in that method as well! - * - * @todo add support to the iterator construct for filtering by a tuple - * revision timestamp range. - */ - public ITupleIterator rangeIterator(// - final byte[] fromKey,// - final byte[] toKey,// - final int capacityIsIgnored,// - final int flags,// - final IFilterConstructor filter// - ) { - - /* - * Does the iterator declare that it will not write back on the index? - */ - final boolean ro = ((flags & IRangeQuery.READONLY) != 0); - - if (ro && ((flags & IRangeQuery.REMOVEALL) != 0)) { - - throw new IllegalArgumentException(); - - } - - /* - * Figure out what base iterator implementation to use. We will layer - * on the optional filter(s) below. - */ - ITupleIterator src; - - if (((flags & REVERSE) == 0) && - ((flags & REMOVEALL) == 0) && - ((flags & CURSOR) == 0)) { - - /* - * Use the recursion-based striterator since it is faster for a - * BTree (but not for an IndexSegment). - * - * Note: The recursion-based striterator does not support remove()! - * - * @todo we could pass in the Tuple here to make the APIs a bit more - * consistent across the recursion-based and the cursor based - * iterators. - * - * @todo when the capacity is one and REVERSE is specified then we - * can optimize this using a reverse traversal striterator - this - * will have lower overhead than the cursor for the BTree (but not - * for an IndexSegment). - */ - src = getRoot().rangeIterator(fromKey, toKey, flags); - - } else { - - final Tuple tuple = new Tuple(this, flags); - - if (isReadOnly()) { - - // Note: this iterator does not allow removal. - src = new ReadOnlyBTreeTupleCursor(((BTree) this), tuple, - fromKey, toKey); - - } else { - - // Note: this iterator supports traversal with concurrent - // modification. - src = new MutableBTreeTupleCursor(((BTree) this), - new Tuple(this, flags), fromKey, toKey); - - } - - if ((flags & REVERSE) != 0) { - - /* - * Reverse scan iterator. - * - * Note: The reverse scan MUST be layered directly over the - * ITupleCursor. Most critically, REMOVEALL combined with a - * REVERSE scan needs to process the tuples in reverse index - * order and then delete them as it goes. - */ - - src = new Reverserator((ITupleCursor) src); - - } - - } - - if (filter != null) { - - /* - * Apply the optional filter. - * - * Note: This needs to be after the reverse scan and before - * REMOVEALL (those are the assumptions for the flags). - */ - - src = filter.newInstance(src); - - } - - if ((flags & REMOVEALL) != 0) { - - assertNotReadOnly(); - - /* - * Note: This iterator removes each tuple that it visits from the - * source iterator. - */ - - src = new TupleRemover() { - @Override - protected boolean remove(ITuple e) { - // remove all visited tuples. - return true; - } - }.filter(src); - - } - - return src; - - } - - /** * Factory for mutable nodes and leaves used by the {@link NodeSerializer}. */ protected static class NodeFactory implements INodeFactory { @@ -1826,6 +1682,7 @@ * Mutable counter. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> + * @version $Id$ */ public static class Counter implements ICounter { @@ -1885,6 +1742,7 @@ * int32 word. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> + * @version $Id$ */ public static class PartitionedCounter implements ICounter { @@ -1993,6 +1851,7 @@ * * @author <a href="mailto:tho...@us...">Bryan * Thompson</a> + * @version $Id$ */ protected static class Stack { @@ -2168,6 +2027,7 @@ * Note: The {@link MutableBTreeTupleCursor} does register such listeners. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> + * @version $Id$ */ public class LeafCursor implements ILeafCursor<Leaf> { @@ -2265,7 +2125,6 @@ } - @Override public LeafCursor clone() { return new LeafCursor(this); @@ -2318,7 +2177,7 @@ } - final public Leaf first() { + public Leaf first() { stack.clear(); @@ -2339,7 +2198,7 @@ } - final public Leaf last() { + public Leaf last() { stack.clear(); @@ -2365,7 +2224,7 @@ * the leaf may not actually contain the key, in which case it is the * leaf that contains the insertion point for the key. */ - final public Leaf seek(final byte[] key) { + public Leaf seek(final byte[] key) { stack.clear(); Copied: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/DelegateIndex.java (from rev 3760, branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/DelegateIndex.java) =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/DelegateIndex.java (rev 0) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/DelegateIndex.java 2010-10-14 20:44:47 UTC (rev 3801) @@ -0,0 +1,169 @@ +/* + +Copyright (C) SYSTAP, LLC 2006-2008. All rights reserved. + +Contact: + SYSTAP, LLC + 4501 Tower Road + Greensboro, NC 27410 + lic...@bi... + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +*/ +/* + * Created on Feb 20, 2008 + */ + +package com.bigdata.btree; + +import com.bigdata.btree.filter.IFilterConstructor; +import com.bigdata.btree.proc.AbstractKeyArrayIndexProcedureConstructor; +import com.bigdata.btree.proc.IKeyRangeIndexProcedure; +import com.bigdata.btree.proc.IResultHandler; +import com.bigdata.btree.proc.ISimpleIndexProcedure; +import com.bigdata.counters.ICounterSet; +import com.bigdata.mdi.IResourceMetadata; + +/** + * An object that delegates its {@link IIndex} interface. + * + * @author <a href="mailto:tho...@us...">Bryan Thompson</a> + * @version $Id$ + */ +public class DelegateIndex implements IIndex { + + private final IIndex delegate; + + /** + * @param delegate + * The delegate. + */ + public DelegateIndex(IIndex delegate) { + + if (delegate == null) { + + throw new IllegalArgumentException(); + + } + + this.delegate = delegate; + + } + + public String toString() { + + final StringBuilder sb = new StringBuilder(); + + sb.append(getClass().getSimpleName()); + + sb.append("{ "); + + sb.append(delegate.toString()); + + sb.append("}"); + + return sb.toString(); + + } + + public boolean contains(byte[] key) { + return delegate.contains(key); + } + + public ICounter getCounter() { + return delegate.getCounter(); + } + + public IndexMetadata getIndexMetadata() { + return delegate.getIndexMetadata(); + } + + public IResourceMetadata[] getResourceMetadata() { + return delegate.getResourceMetadata(); + } + + public ICounterSet getCounters() { + return delegate.getCounters(); + } + + public byte[] insert(byte[] key, byte[] value) { + return delegate.insert(key, value); + } + + public byte[] lookup(byte[] key) { + return delegate.lookup(key); + } + + public long rangeCount() { + return delegate.rangeCount(); + } + + public long rangeCount(byte[] fromKey, byte[] toKey) { + return delegate.rangeCount(fromKey, toKey); + } + + public long rangeCountExact(byte[] fromKey, byte[] toKey) { + return delegate.rangeCountExact(fromKey, toKey); + } + + public long rangeCountExactWithDeleted(byte[] fromKey, byte[] toKey) { + return delegate.rangeCountExactWithDeleted(fromKey, toKey); + } + + public ITupleIterator rangeIterator() { + return rangeIterator(null,null); + } + + public ITupleIterator rangeIterator(byte[] fromKey, byte[] toKey, int capacity, int flags, IFilterConstructor filter) { + return delegate.rangeIterator(fromKey, toKey, capacity, flags, filter); + } + + public ITupleIterator rangeIterator(byte[] fromKey, byte[] toKey) { + return delegate.rangeIterator(fromKey, toKey); + } + + public byte[] remove(byte[] key) { + return delegate.remove(key); + } + + public Object submit(byte[] key, ISimpleIndexProcedure proc) { + return delegate.submit(key, proc); + } + + public void submit(byte[] fromKey, byte[] toKey, IKeyRangeIndexProcedure proc, IResultHandler handler) { + delegate.submit(fromKey, toKey, proc, handler); + } + + public void submit(int fromIndex, int toIndex, byte[][] keys, byte[][] vals, AbstractKeyArrayIndexProcedureConstructor ctor, IResultHandler handler) { + delegate.submit(fromIndex, toIndex, keys, vals, ctor, handler); + } + + public boolean contains(Object key) { + return delegate.contains(key); + } + + public Object insert(Object key, Object value) { + return delegate.insert(key, value); + } + + public Object lookup(Object key) { + return delegate.lookup(key); + } + + public Object remove(Object key) { + return delegate.remove(key); + } + +} Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/IndexSegment.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/IndexSegment.java 2010-10-14 16:19:22 UTC (rev 3800) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/IndexSegment.java 2010-10-14 20:44:47 UTC (rev 3801) @@ -29,20 +29,15 @@ import com.bigdata.btree.IndexSegment.ImmutableNodeFactory.ImmutableLeaf; import com.bigdata.btree.data.ILeafData; import com.bigdata.btree.data.INodeData; -import com.bigdata.btree.filter.IFilterConstructor; -import com.bigdata.btree.filter.Reverserator; -import com.bigdata.btree.filter.TupleRemover; import com.bigdata.btree.raba.IRaba; import com.bigdata.btree.raba.ReadOnlyKeysRaba; import com.bigdata.btree.raba.ReadOnlyValuesRaba; import com.bigdata.io.AbstractFixedByteArrayBuffer; -import com.bigdata.io.DirectBufferPool; import com.bigdata.io.FixedByteArrayBuffer; import com.bigdata.mdi.IResourceMetadata; import com.bigdata.service.Event; import com.bigdata.service.EventResource; import com.bigdata.service.EventType; -import org.apache.log4j.Logger; /** * An index segment is read-only btree corresponding to some key range of a @@ -56,15 +51,11 @@ * leaves will all refuse mutation operations). * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> + * @version $Id$ */ public class IndexSegment extends AbstractBTree { /** - * Log for btree opeations. - */ - private static final Logger log = Logger.getLogger(IndexSegment.class); - - /** * Type safe reference to the backing store. */ private final IndexSegmentStore fileStore; @@ -169,7 +160,6 @@ * {@link #getEntryCount()} uses the {@link IndexSegmentCheckpoint} and that * is only available while the {@link IndexSegmentStore} is open. */ - @Override public String toString() { // make sure the fileStore will remain open. @@ -313,7 +303,7 @@ } @Override - final protected void _reopen() { + protected void _reopen() { // prevent concurrent close. fileStore.lock.lock(); @@ -694,138 +684,6 @@ }; } - /** - * Core implementation. - * <p> - * Note: If {@link IRangeQuery#CURSOR} is specified the returned iterator - * supports traversal with concurrent modification by a single-threaded - * process (the {@link BTree} is NOT thread-safe for writers). Write are - * permitted iff {@link AbstractBTree} allows writes. - * <p> - * Note: {@link IRangeQuery#REVERSE} is handled here by wrapping the - * underlying {@link ITupleCursor}. - * <p> - * Note: {@link IRangeQuery#REMOVEALL} is handled here by wrapping the - * iterator. - * <p> - * Note: - * {@link FusedView#rangeIterator(byte[], byte[], int, int, IFilterConstructor)} - * is also responsible for constructing an {@link ITupleIterator} in a - * manner similar to this method. If you are updating the logic here, then - * check the logic in that method as well! - * - * @todo add support to the iterator construct for filtering by a tuple - * revision timestamp range. - */ - public ITupleIterator rangeIterator(// - final byte[] fromKey,// - final byte[] toKey,// - final int capacityIsIgnored,// - final int flags,// - final IFilterConstructor filter// - ) { - - /* - * Does the iterator declare that it will not write back on the index? - */ - final boolean readOnly = ((flags & IRangeQuery.READONLY) != 0); - - if (readOnly && ((flags & IRangeQuery.REMOVEALL) != 0)) { - - throw new IllegalArgumentException(); - - } - - /* - * Figure out what base iterator implementation to use. We will layer - * on the optional filter(s) below. - */ - ITupleIterator src; - - final Tuple tuple = new Tuple(this, flags); - - final IndexSegment seg = (IndexSegment) this; - - /* - * @todo we could scan the list of pools and chose the best fit - * pool and then allocate a buffer from that pool. Best fit - * would mean either the byte range fits without "too much" slop - * or the #of reads will have to perform is not too large. We - * might also want to limit the maximum size of the reads. - */ - -// final DirectBufferPool pool = DirectBufferPool.INSTANCE_10M; - final DirectBufferPool pool = DirectBufferPool.INSTANCE; - - if (true - && ((flags & REVERSE) == 0) - && ((flags & CURSOR) == 0) - && (seg.getStore().getCheckpoint().maxNodeOrLeafLength <= pool - .getBufferCapacity()) - && ((rangeCount(fromKey, toKey) / branchingFactor) > 2)) { - - src = new IndexSegmentMultiBlockIterator(seg, pool, - fromKey, toKey, flags); - - } else { - - src = new IndexSegmentTupleCursor(seg, tuple, fromKey, toKey); - - } - - - if ((flags & REVERSE) != 0) { - - /* - * Reverse scan iterator. - * - * Note: The reverse scan MUST be layered directly over the - * ITupleCursor. Most critically, REMOVEALL combined with a - * REVERSE scan needs to process the tuples in reverse index - * order and then delete them as it goes. - */ - - src = new Reverserator((ITupleCursor) src); - - } - - - if (filter != null) { - - /* - * Apply the optional filter. - * - * Note: This needs to be after the reverse scan and before - * REMOVEALL (those are the assumptions for the flags). - */ - - src = filter.newInstance(src); - - } - - if ((flags & REMOVEALL) != 0) { - - assertNotReadOnly(); - - /* - * Note: This iterator removes each tuple that it visits from the - * source iterator. - */ - - src = new TupleRemover() { - @Override - protected boolean remove(ITuple e) { - // remove all visited tuples. - return true; - } - }.filter(src); - - } - - return src; - - } - /* * INodeFactory */ @@ -861,6 +719,7 @@ * * @author <a href="mailto:tho...@us...">Bryan * Thompson</a> + * @version $Id$ */ public static class ImmutableNode extends Node { @@ -909,6 +768,8 @@ * * @author <a href="mailto:tho...@us...">Bryan * Thompson</a> + * @version $Id: IndexSegment.java 2265 2009-10-26 12:51:06Z thompsonbry + * $ */ private static class EmptyReadOnlyLeafData implements ILeafData { @@ -1008,6 +869,7 @@ * * @author <a href="mailto:tho...@us...">Bryan * Thompson</a> + * @version $Id$ */ public static class ImmutableLeaf extends Leaf { @@ -1145,6 +1007,7 @@ // * // * @author <a href="mailto:tho...@us...">Bryan // * Thompson</a> +// * @version $Id$ // */ // static public class ImmutableEmptyLastLeaf extends ImmutableLeaf { // @@ -1183,6 +1046,7 @@ * A position for the {@link IndexSegmentTupleCursor}. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> + * @version $Id$ * @param <E> * The generic type for objects de-serialized from the values in * the index. @@ -1190,7 +1054,6 @@ static private class CursorPosition<E> extends AbstractCursorPosition<ImmutableLeaf,E> { @SuppressWarnings("unchecked") - @Override public IndexSegmentTupleCursor<E> getCursor() { return (IndexSegmentTupleCursor)cursor; @@ -1239,6 +1102,7 @@ * listeners for concurrent modifications. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> + * @version $Id$ * @param <E> * The generic type for the objects de-serialized from the index. */ @@ -1401,6 +1265,7 @@ * Cursor using the double-linked leaves for efficient scans. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> + * @version $Id$ */ public class ImmutableLeafCursor implements ILeafCursor<ImmutableLeaf> { @@ -1418,7 +1283,6 @@ } - @Override public ImmutableLeafCursor clone() { return new ImmutableLeafCursor(this); @@ -1469,7 +1333,7 @@ } - final public ImmutableLeaf seek(final byte[] key) { + public ImmutableLeaf seek(final byte[] key) { leaf = findLeaf(key); @@ -1499,7 +1363,7 @@ } - final public ImmutableLeaf first() { + public ImmutableLeaf first() { final long addr = getStore().getCheckpoint().addrFirstLeaf; @@ -1509,7 +1373,7 @@ } - final public ImmutableLeaf last() { + public ImmutableLeaf last() { final long addr = getStore().getCheckpoint().addrLastLeaf; Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/MetadataIndex.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/MetadataIndex.java 2010-10-14 16:19:22 UTC (rev 3800) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/MetadataIndex.java 2010-10-14 20:44:47 UTC (rev 3801) @@ -29,13 +29,13 @@ import java.io.ObjectOutput; import java.util.UUID; +import com.bigdata.mdi.PartitionLocator; import org.CognitiveWeb.extser.LongPacker; import com.bigdata.btree.keys.IKeyBuilderFactory; import com.bigdata.btree.view.FusedView; import com.bigdata.journal.ICommitter; import com.bigdata.journal.IResourceManager; -import com.bigdata.mdi.PartitionLocator; import com.bigdata.rawstore.IRawStore; import com.bigdata.service.MetadataService; @@ -44,7 +44,7 @@ * metadata index for each distributed index. The keys of the metadata index are * the first key that would be directed into the corresponding index segment, * e.g., a <em>separator key</em> (this is just the standard btree semantics). - * The values are serialized {@link PartitionLocator} objects. + * The values are serialized {@link com.bigdata.mdi.PartitionLocator} objects. * <p> * Note: At this time the recommended scale-out approach for the metadata index * is to place the metadata indices on a {@link MetadataService} (the same @@ -63,6 +63,7 @@ * taking the database offline. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> + * @version $Id$ * * @todo The {@link MetadataIndex} does NOT support either overflow (it may NOT * be a {@link FusedView}) NOR key-range splits. There are several issues @@ -96,7 +97,6 @@ */ private transient final MetadataIndexView view; - @Override public MetadataIndexMetadata getIndexMetadata() { return (MetadataIndexMetadata) super.getIndexMetadata(); @@ -210,7 +210,6 @@ * Extended to require a checkpoint if {@link #incrementAndGetNextPartitionId()} has been * invoked. */ - @Override public boolean needsCheckpoint() { if(nextPartitionId != ((MetadataIndexCheckpoint)getCheckpoint()).getNextPartitionId()) { @@ -228,6 +227,7 @@ * identifier to be assigned by the metadata index. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> + * @version $Id$ */ public static class MetadataIndexCheckpoint extends Checkpoint { @@ -329,6 +329,7 @@ * for the managed scale-out index. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> + * @version $Id$ */ public static class MetadataIndexMetadata extends IndexMetadata implements Externalizable { @@ -380,7 +381,6 @@ private static final transient int VERSION0 = 0x0; - @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { @@ -398,7 +398,6 @@ } - @Override public void writeExternal(ObjectOutput out) throws IOException { super.writeExternal(out); @@ -431,6 +430,7 @@ * {@link MetadataIndex}. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> + * @version $Id$ */ public static class PartitionLocatorTupleSerializer extends DefaultTupleSerializer<byte[]/*key*/, PartitionLocator/*val*/> { @@ -473,7 +473,6 @@ */ private final static transient byte VERSION = VERSION0; - @Override public void readExternal(final ObjectInput in) throws IOException, ClassNotFoundException { @@ -491,7 +490,6 @@ } - @Override public void writeExternal(final ObjectOutput out) throws IOException { super.writeExternal(out); Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/MetadataIndexView.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/MetadataIndexView.java 2010-10-14 16:19:22 UTC (rev 3800) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/MetadataIndexView.java 2010-10-14 20:44:47 UTC (rev 3801) @@ -28,27 +28,27 @@ package com.bigdata.btree; -import com.bigdata.btree.MetadataIndex.MetadataIndexMetadata; -import com.bigdata.btree.filter.IFilterConstructor; +import com.bigdata.mdi.PartitionLocator; import org.apache.log4j.Logger; import com.bigdata.cache.LRUCache; -import com.bigdata.mdi.PartitionLocator; +import com.bigdata.btree.MetadataIndex.MetadataIndexMetadata; /** * The extension semantics for the {@link IMetadataIndex} are implemented by * this class. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> + * @version $Id$ */ -public class MetadataIndexView implements IMetadataIndex { +public class MetadataIndexView extends DelegateIndex implements IMetadataIndex { protected static final Logger log = Logger.getLogger(MetadataIndexView.class); // protected static final boolean INFO = log.isInfoEnabled(); // protected static final boolean DEBUG = log.isDebugEnabled(); - private final MetadataIndex metadataIndex; + private final AbstractBTree delegate; /** * <code>true</code> iff this is a read-only view. this is used to @@ -58,14 +58,19 @@ */ final private boolean readOnly; - public MetadataIndexView(MetadataIndex metadataIndex) { - this.metadataIndex = metadataIndex; - this.readOnly = metadataIndex.isReadOnly(); + public MetadataIndexView(AbstractBTree delegate) { + + super(delegate); + + this.delegate = delegate; + + this.readOnly = delegate.isReadOnly(); + } public MetadataIndexMetadata getIndexMetadata() { - return (MetadataIndexMetadata) metadataIndex.getIndexMetadata(); + return (MetadataIndexMetadata) super.getIndexMetadata(); } @@ -82,7 +87,7 @@ * de-serialization using the ITupleSerializer. */ - return (PartitionLocator) metadataIndex.lookup((Object) key); + return (PartitionLocator) delegate.lookup((Object) key); } @@ -107,7 +112,7 @@ */ private PartitionLocator find_with_iterator(byte[] key) { - final ITupleIterator<PartitionLocator> itr = metadataIndex.rangeIterator( + final ITupleIterator<PartitionLocator> itr = delegate.rangeIterator( null/* fromKey */, key/* toKey */, 1/* capacity */, IRangeQuery.VALS | IRangeQuery.REVERSE, null/* filter */); @@ -142,7 +147,7 @@ if (key == null) { // use the index of the last partition. - index = metadataIndex.getEntryCount() - 1; + index = delegate.getEntryCount() - 1; } else { @@ -207,7 +212,7 @@ /** * Remove the locator from the {@link #locatorCache}. It will be re-read on - * demand from the {@link #metadataIndex}. + * demand from the {@link #delegate}. */ public void staleLocator(PartitionLocator locator) { @@ -225,8 +230,8 @@ */ private PartitionLocator getLocatorAtIndex(int index) { - final ITuple<PartitionLocator> tuple = metadataIndex.valueAt(index, - metadataIndex.getLookupTuple()); + final ITuple<PartitionLocator> tuple = delegate.valueAt(index, + delegate.getLookupTuple()); return tuple.getObject(); @@ -247,7 +252,7 @@ */ private int findIndexOf(byte[] key) { - int pos = metadataIndex.indexOf(key); + int pos = delegate.indexOf(key); if (pos < 0) { @@ -261,7 +266,7 @@ if(pos == 0) { - if(metadataIndex.getEntryCount() != 0) { + if(delegate.getEntryCount() != 0) { throw new IllegalStateException( "Partition not defined for empty key."); @@ -289,34 +294,4 @@ } - public long rangeCount() { - return metadataIndex.rangeCount(); - } - - public long rangeCount(byte[] fromKey, byte[] toKey) { - return metadataIndex.rangeCount(fromKey, toKey); - } - - public long rangeCountExact(byte[] fromKey, byte[] toKey) { - return metadataIndex.rangeCountExact(fromKey, toKey); - } - - public long rangeCountExactWithDeleted(byte[] fromKey, byte[] toKey) { - return metadataIndex.rangeCountExactWithDeleted(fromKey, toKey); - } - - public ITupleIterator rangeIterator() { - return metadataIndex.rangeIterator(); - } - - public ITupleIterator rangeIterator(byte[] fromKey, byte[] toKey) { - return metadataIndex.rangeIterator(fromKey, toKey); - } - - public ITupleIterator rangeIterator(byte[] fromKey, byte[] toKey, - int capacity, int flags, - IFilterConstructor filterCtor) { - return metadataIndex.rangeIterator(fromKey, toKey, capacity, flags, filterCtor); - } - } Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/Node.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/Node.java 2010-10-14 16:19:22 UTC (rev 3800) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/Node.java 2010-10-14 20:44:47 UTC (rev 3801) @@ -56,7 +56,6 @@ import cutthecrap.utils.striterators.Expander; import cutthecrap.utils.striterators.SingleValueIterator; import cutthecrap.utils.striterators.Striterator; -import org.apache.log4j.Logger; /** * <p> @@ -83,15 +82,11 @@ * we can prune the search before we materialize the child. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> + * @version $Id$ */ public class Node extends AbstractNode<Node> implements INodeData { /** - * Log for btree opeations. - */ - private static final Logger log = Logger.getLogger(Node.class); - - /** * The data record. {@link MutableNodeData} is used for all mutation * operations. {@link ReadOnlyNodeData} is used when the {@link Node} is * made persistent. A read-only data record is automatically converted into @@ -628,7 +623,7 @@ btree.getBtreeCounters().rootsSplit++; - if (log.isInfoEnabled() || BigdataStatics.debug) { + if (BTree.log.isInfoEnabled() || BigdataStatics.debug) { // Note: nnodes and nleaves might not reflect rightSibling yet. @@ -637,8 +632,8 @@ + ", m=" + btree.getBranchingFactor() + ", nentries=" + btree.nentries; - if (log.isInfoEnabled()) - log.info(msg); + if (BTree.log.isInfoEnabled()) + BTree.log.info(msg); if (BigdataStatics.debug) System.err.println(msg); @@ -2405,8 +2400,8 @@ // one less node in the tree. btree.nnodes--; - if (INFO) { - log.info("reduced tree height: height=" + if (BTree.INFO) { + BTree.log.info("reduced tree height: height=" + btree.height + ", newRoot=" + btree.root); } Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/ReadOnlyIndex.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/ReadOnlyIndex.java 2010-10-14 16:19:22 UTC (rev 3800) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/ReadOnlyIndex.java 2010-10-14 20:44:47 UTC (rev 3801) @@ -27,9 +27,6 @@ package com.bigdata.btree; -import com.bigdata.btree.proc.IKeyRangeIndexProcedure; -import com.bigdata.btree.proc.ISimpleIndexProcedure; -import com.bigdata.counters.ICounterSet; import java.util.Iterator; import com.bigdata.btree.filter.IFilterConstructor; @@ -50,19 +47,20 @@ * @see {@link IResourceManager#getIndex(String, long)} * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> + * @version $Id$ */ -public class ReadOnlyIndex implements IIndex { - - private IIndex src; +public class ReadOnlyIndex extends DelegateIndex { public ReadOnlyIndex(IIndex src) { - this.src = src; + + super(src); + } /** {@link IndexMetadata} is cloned to disallow modification. */ final public IndexMetadata getIndexMetadata() { - return src.getIndexMetadata().clone(); + return super.getIndexMetadata().clone(); } @@ -73,7 +71,7 @@ */ final public IResourceMetadata[] getResourceMetadata() { - return src.getResourceMetadata().clone(); + return super.getResourceMetadata().clone(); } @@ -82,7 +80,7 @@ */ final public ICounter getCounter() { - return new ReadOnlyCounter(src.getCounter()); + return new ReadOnlyCounter(super.getCounter()); } @@ -123,7 +121,7 @@ /* * Must explicitly disable Iterator#remove(). */ - return new ReadOnlyEntryIterator(src.rangeIterator(fromKey, toKey, + return new ReadOnlyEntryIterator(super.rangeIterator(fromKey, toKey, capacity, flags, filter)); } @@ -175,65 +173,4 @@ } - public ICounterSet getCounters() { - return src.getCounters(); - } - - public Object submit(byte[] key, ISimpleIndexProcedure proc) { - return src.submit(key, proc); - } - - public void submit(byte[] fromKey, byte[] toKey, - IKeyRangeIndexProcedure proc, IResultHandler handler) { - src.submit(fromKey, toKey, proc, handler); - } - - public byte[] lookup(byte[] key) { - return src.lookup(key); - } - - public boolean contains(byte[] key) { - return src.contains(key); - } - - public Object insert(Object key, Object value) { - return src.insert(key, value); - } - - public Object lookup(Object key) { - return src.lookup(key); - } - - public boolean contains(Object key) { - return src.contains(key); - } - - public Object remove(Object key) { - return src.remove(key); - } - - public long rangeCount() { - return src.rangeCount(); - } - - public long rangeCount(byte[] fromKey, byte[] toKey) { - return src.rangeCount(fromKey, toKey); - } - - public long rangeCountExact(byte[] fromKey, byte[] toKey) { - return src.rangeCountExact(fromKey, toKey); - } - - public long rangeCountExactWithDeleted(byte[] fromKey, byte[] toKey) { - return src.rangeCountExactWithDeleted(fromKey, toKey); - } - - public ITupleIterator rangeIterator() { - return src.rangeIterator(); - } - - public ITupleIterator rangeIterator(byte[] fromKey, byte[] toKey) { - return src.rangeIterator(fromKey, toKey); - } - } Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRURecycler.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRURecycler.java 2010-10-14 16:19:22 UTC (rev 3800) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRURecycler.java 2010-10-14 20:44:47 UTC (rev 3801) @@ -618,7 +618,6 @@ e.prior = last; last = e; } - //System.err.println("++size"); size++; counters.bytesInMemory.addAndGet(e.bytesInMemory); counters.bytesOnDisk.addAndGet(e.bytesOnDisk); @@ -660,7 +659,6 @@ e.cache = null; // clear reference to the cache. e.k = null; // clear the key. e.v = null; // clear the value reference. - //System.err.println("--size"); size--; counters.bytesInMemory.addAndGet(-e.bytesInMemory); counters.bytesOnDisk.addAndGet(-e.bytesOnDisk); Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/LRUNexus.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/LRUNexus.java 2010-10-14 16:19:22 UTC (rev 3800) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/LRUNexus.java 2010-10-14 20:44:47 UTC (rev 3801) @@ -47,6 +47,7 @@ * system properties defined by {@link Options}. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> + * @version $Id$ * * FIXME LRUNexus : writes MUST must be "isolated" until the commit. * Isolated indices MUST have their own cache backed by the shared LRU @@ -130,6 +131,7 @@ * * @author <a href="mailto:tho...@us...">Bryan * Thompson</a> + * @version $Id$ */ public static interface Options { @@ -341,13 +343,26 @@ } + /** + * Global instance. + * <p> + * Note: A <a href="http://bugs.sun.com/view_bug.do?bug_id=6880903">Sun G1 + * bug in JDK 1.6.0_16</a> provides a false estimate of the available + * memory. + * + * @see Options + */ + public static final IGlobalLRU<Long, Object> INSTANCE; + + /** * The access policy. Not all {@link IGlobalLRU} implementations support * multiple access policies. Check the specific implementation to see which * policies it supports. * * @author <a href="mailto:tho...@us...">Bryan * Thompson</a> + * @version $Id$ */ public static enum AccessPolicyEnum { @@ -380,6 +395,7 @@ * * @author <a href="mailto:tho...@us...">Bryan * Thompson</a> + * @version $Id$ */ public static class CacheSettings { @@ -531,8 +547,9 @@ Options.INDEX_SEGMENT_BUILD_POPULATES_CACHE, Options.DEFAULT_INDEX_SEGMENT_BUILD_POPULATES_CACHE)); ... [truncated message content] |