Menu
▾
▴
bigdata-commit
|
From: <btm...@us...> - 2010-07-23 12:19:21
|
Revision: 3269
http://bigdata.svn.sourceforge.net/bigdata/?rev=3269&view=rev
Author: btmurphy
Date: 2010-07-23 12:19:14 +0000 (Fri, 23 Jul 2010)
Log Message:
-----------
[branch dev-btm]: CHECKPOINT - smart proxy work, still incomplete and including debug println's and BTM markers, but checkpointed for syncing with trunk changes
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractCounterSet.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/LocalDataServiceFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/util/config/NicUtil.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/StressTestConcurrent.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/AbstractServicesManagerService.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ManageLogicalServiceTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/MonitorCreatePhysicalServiceLocksTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/RestartPersistentServices.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServiceConfigurationZNodeMonitorTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServicesManagerServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServicesManagerStartupTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/AbstractHostConstraint.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/BigdataServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/JavaServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/JiniCoreServicesConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/JiniServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/LoadBalancerConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ManagedServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServicesManagerConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ZookeeperServerEntry.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/JiniCoreServicesProcessHelper.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/JiniServiceProcessHelper.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/ProcessHelper.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/ZookeeperProcessHelper.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/EmbeddedLoadBalancer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/ServiceProxy.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/config/logging.properties
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/AbstractServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/JiniFederation.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/lookup/LoadBalancerClient.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/JiniServicesHelper.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/LookupStarter.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/config/lookup.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/Util.java
branches/dev-btm/bigdata-jini/src/resources/config/bigdataStandaloneTesting.config
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/config/TestZookeeperServerEntry.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/testReggie.config
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/testStartJini.config
branches/dev-btm/bigdata-jini/src/test/com/bigdata/zookeeper/testzoo.config
branches/dev-btm/build.properties
branches/dev-btm/build.xml
branches/dev-btm/src/resources/config/bigdataCluster.config
branches/dev-btm/src/resources/config/jini/reggie.config
branches/dev-btm/src/resources/config/log4j.properties
branches/dev-btm/src/resources/config/logging.properties
Added Paths:
-----------
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/EntryUtil.java
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractCounterSet.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractCounterSet.java 2010-07-23 11:04:00 UTC (rev 3268)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractCounterSet.java 2010-07-23 12:19:14 UTC (rev 3269)
@@ -42,6 +42,8 @@
*/
public abstract class AbstractCounterSet implements ICounterSet {
+public static org.apache.log4j.Logger logger = com.bigdata.util.config.LogUtil.getLog4jLogger((AbstractCounterSet.class).getName());
+
protected final String name;
protected AbstractCounterSet parent;
@@ -198,11 +200,39 @@
if( path.contains("//")) {
+if (logger.isEnabledFor(org.apache.log4j.Level.WARN)) {
+ logger.warn("***** AbstractCounterSet.getPath: CONTAINS SLASH-SLASH: path INPUT = "+path);
+ logger.warn("***** AbstractCounterSet.getPath: path.length = "+path.length());
+ byte[] pathBytes = path.getBytes();
+ StringBuffer strBuf = new StringBuffer();
+ if( (pathBytes[0] < 32) || (pathBytes[0] > 126) ) {
+ strBuf.append("X");
+ }else{
+ strBuf.append( new String(new byte[] {pathBytes[0]}) );
+ }
+ for(int i=1;i<pathBytes.length; i++) {
+ if( (pathBytes[i] < 32) || (pathBytes[i] > 126) ) {
+ strBuf.append("X");
+ }else{
+ strBuf.append( new String(new byte[] {pathBytes[i]}) );
+ }
+ }
+ logger.warn("***** AbstractCounterSet.getPath: CONTAINS SLASH-SLASH: path CONVERTED = "+strBuf.toString());
+}
+int slashSlashIndex = path.indexOf("//");
+if(slashSlashIndex >= 0) {//contains "//"
+ logger.warn("***** AbstractCounterSet.getPath: INDEX OF SLASH-SLASH = "+slashSlashIndex);
+}else{//does not contain "//"
+ logger.warn("***** AbstractCounterSet.getPath: !!!! path CONTAINS SLASH-SLASH, but indexOf = "+slashSlashIndex+" !!!!");
+}
+
+if(slashSlashIndex >= 0) {//throw exception only if BOTH path.contains AND path.indexOf say that the given path contains "//"
/*
* Empty path names are not allowed.
*/
throw new IllegalArgumentException(path);
+}//BTM
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-07-23 11:04:00 UTC (rev 3268)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-07-23 12:19:14 UTC (rev 3269)
@@ -1388,7 +1388,7 @@
// Note: This _is_ a local method call.
final UUID serviceUUID = fed.getServiceUUID();
-
+System.out.println("\n>>>>> AbstractFederation.reportPerformanceCounters: serviceUUID = "+serviceUUID);
// Will be null until assigned by the service registrar.
if (serviceUUID == null) {
@@ -1403,12 +1403,14 @@
final LoadBalancer loadBalancerService = fed.getLoadBalancerService();
if (loadBalancerService == null) {
+System.out.println(">>>>> AbstractFederation.reportPerformanceCounters: loadBalancerService = NULL");
log.warn("Could not discover load balancer service.");
return;
}
+System.out.println(">>>>> AbstractFederation.reportPerformanceCounters: loadBalancerService = "+loadBalancerService);
/*
* @todo this is probably worth compressing as there will be a lot
@@ -1419,9 +1421,12 @@
final ByteArrayOutputStream baos = new ByteArrayOutputStream(
Bytes.kilobyte32 * 2);
+System.out.println(">>>>> AbstractFederation.reportPerformanceCounters: CALLING fed.getCounterSet() ...");
fed.getCounterSet().asXML(baos, "UTF-8", null/* filter */);
+System.out.println(">>>>> AbstractFederation.reportPerformanceCounters: CALLING loadBalancer.notify ...");
loadBalancerService.notify(serviceUUID, baos.toByteArray());
+System.out.println(">>>>> AbstractFederation.reportPerformanceCounters: DONE CALLING loadBalancer.notify");
if (log.isInfoEnabled())
log.info("Notified the load balancer.");
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedFederation.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedFederation.java 2010-07-23 11:04:00 UTC (rev 3268)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedFederation.java 2010-07-23 12:19:14 UTC (rev 3269)
@@ -620,6 +620,9 @@
//BTM p.setProperty(LoadBalancerService.Options.TRANSIENT, "true");
p.setProperty(EmbeddedLoadBalancerServiceImpl.Options.TRANSIENT, "true");
+p.setProperty(EmbeddedLoadBalancerServiceImpl.Options.LOG_DIR,
+ new File(EmbeddedLoadBalancerServiceImpl.Options.DEFAULT_LOG_DIR).toString());
+
} else {
// specify the data directory for the load balancer.
@@ -860,6 +863,7 @@
{
super(serviceUUID, hostname,
sdm,
+properties.getProperty(EmbeddedLoadBalancerServiceImpl.Options.LOG_DIR),
//BTM*** EmbeddedFederation.this,
dataServiceMap,//BTM*** - remove after DataService smart proxy?
properties);
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/LocalDataServiceFederation.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/LocalDataServiceFederation.java 2010-07-23 11:04:00 UTC (rev 3268)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/LocalDataServiceFederation.java 2010-07-23 12:19:14 UTC (rev 3269)
@@ -193,6 +193,7 @@
loadBalancerService = new EmbeddedLoadBalancer(UUID.randomUUID(),
hostname,
null,//SDM - replace with real SDM after conversion to smart proxy?
+p.getProperty(EmbeddedLoadBalancer.Options.LOG_DIR),
//BTM*** LocalDataServiceFederation.this,
//BTM*** remove after EmbeddedDataService is converted to smart proxy?
dataServiceMap,
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/util/config/NicUtil.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/util/config/NicUtil.java 2010-07-23 11:04:00 UTC (rev 3268)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/util/config/NicUtil.java 2010-07-23 12:19:14 UTC (rev 3269)
@@ -505,6 +505,253 @@
}
/**
+ * Special-purpose convenience method that returns a
+ * <code>String</code> value representing the ip address of
+ * the current node; where the value that is returned is
+ * determined according to following criteria:
+ * <p>
+ * <ul>
+ * <li> If a non-<code>null</code> value is input for the
+ * <code>systemPropertyName</code> parameter, then
+ * this is viewed as a declaration by the caller that
+ * that the system property with that given value
+ * should take precedence over all other means of
+ * determining the desired ip address. As such, this
+ * method determines if a system property having the
+ * given has indeed been set and, if it has, returns
+ * the ip address of the nic having that name; or
+ * <code>null</code> if there is no nic with the
+ * desired name installed on the node.
+ * <li> If a non-<code>null</code> value is input for the
+ * <code>systemPropertyName</code> parameter, but
+ * no system property with that name has been set,
+ * and <code>true</code> has been passed in for
+ * the <code>fallbackOk</code> parameter, then this
+ * method will return the IPV4 based address of the
+ * first reachable nic that is found on the node.
+ * Upon failing to find such an address, if the
+ * <code>loopbackOk</code> parameter is also
+ * <code>true</code>, then this method will return
+ * the <i>loop back</i> address of the node; otherwise
+ * <code>null</code> is returned.
+ * <li> If <code>null</code> is input for the
+ * <code>systemPropertyName</code> parameter, but
+ * a non-<code>null</code> value is input for the
+ * <code>defaultNic</code> parameter, then this
+ * method returns the ip address of the nic having
+ * that name; or <code>null</code> if there is no
+ * nic with the desired default name installed on the
+ * node.
+ * <li> If <code>null</code> is input for both the
+ * <code>systemPropertyName</code> parameter and the
+ * <code>defaultNic</code> parameter, and if the
+ * <code>fallbackOk</code> parameter is <code>true</code>,
+ * then this method will return the IPV4 based address
+ * of the first reachable nic that is found on the node.
+ * Upon failing to find such an address, if the
+ * <code>loopbackOk</code> parameter is also
+ * <code>true</code>, then this method will return
+ * the <i>loop back</i> address of the node; otherwise
+ * <code>null</code> is returned.
+ * </ul>
+ * <p>
+ * This method can be called from within a configuration
+ * as well as from within program control.
+ *
+ * @param systemPropertyName <code>String</code> value containing
+ * the name of a system property whose
+ * value is the network interface name
+ * whose ip address should be returned.
+ * May be <code>null</code>.
+ *
+ * @param defaultNic <code>String</code> value containing
+ * the name of the network interface
+ * whose ip address should be returned
+ * if <code>null</code> is input for the
+ * <code>systemPropertyName</code> parameter.
+ *
+ * @param fallbackOk if <code>true</code>, then if either
+ * no system property is set having the
+ * name referenced by the
+ * <code>systemPropertyName</code> parameter,
+ * or if <code>null</code> is input for both
+ * the <code>systemPropertyName</code>
+ * parameter and the <code>defaultNic</code>
+ * parameter, return the IPV4 based address
+ * of the first reachable network interface
+ * that can be found on the node.
+ *
+ * @param loopbackOk if <code>true</code>, and if <code>true</code>
+ * is also input for the <code>fallbackOk</code>
+ * parameter, then if this method attempts,
+ * but fails, to find a valid IPV4 fallback
+ * address, then the node's <i>loop back</i>
+ * address is returned.
+ *
+ * @return a <code>String</code> representing an ip address associated
+ * with the current node; where the value that is returned is
+ * determined according to the criteria described above.
+ */
+ public static String getIpAddress(String systemPropertyName,
+ String defaultNic,
+ boolean fallbackOk,
+ boolean loopbackOk)
+ throws SocketException, IOException
+ {
+ if(systemPropertyName != null) {//system property takes precedence
+ String nicName = System.getProperty(systemPropertyName);
+ boolean propSet = true;
+ if(nicName == null) {
+ propSet = false;
+ } else {
+ // handle ant script case where the system property
+ // may not have been set on the command line, but
+ // was still set to "${<systemPropertyName>}" using
+ // ant <sysproperty> tag
+ String rawProp = "${" + systemPropertyName + "}";
+ if( rawProp.equals(nicName) ) propSet = false;
+ }
+ if(propSet) {
+ return getIpAddress(nicName, 0, loopbackOk);
+ } else {//desired system property not set, try fallback
+ if(fallbackOk) {
+ return getDefaultIpv4Address(loopbackOk);
+ } else {
+ return null;
+ }
+ }
+ } else {//no system property name provided, try default nic
+ if(defaultNic != null) {
+ return getIpAddress(defaultNic, 0, loopbackOk);
+ } else {//no default nic provided, try fallback
+ if(fallbackOk) {
+ return getDefaultIpv4Address(loopbackOk);
+ } else {
+ return null;
+ }
+ }
+ }
+ }
+
+ public static String getIpAddress()
+ throws SocketException, IOException
+ {
+ return getIpAddress(null, null, true, true);
+ }
+
+ /**
+ * Examines each address associated with each network interface
+ * card (nic) installed on the current node, and returns the
+ * <code>String</code> value of the first such address that is
+ * determined to be both <i>reachable</i> and an address type
+ * that represents an <i>IPv4</i> address.
+ *
+ * This method will always first examine addresses that are
+ * <i>not</i> the <i>loopback</i> address (<i>local host</i>);
+ * returning a loopback adddress only if <code>true</code>
+ * is input for the <code>loopbackOk</code> parameter, and
+ * none of the non-loopback addresses satisfy this method's
+ * search criteria.
+ *
+ * If this method fails to find any address that satisfies the
+ * above criteria, then this method returns <code>null</code>.
+ *
+ * @param loopbackOk if <code>true</code>, then upon failure
+ * find an non-<i>loopback</i> address that
+ * satisfies this method's search criteria
+ * (an IPv4 type address and reachable), the
+ * first loopback address that is found to be
+ * reachable is returned.
+ *
+ * If <code>false</code> is input for this
+ * parameter, then this method will examine
+ * only those addresses that do <i>not</i>
+ * correspond to the corresponding nic's
+ * loopback address.
+ *
+ * @return a <code>String</code> value representing the first
+ * network interface address installed on the current
+ * node that is determined to be both <i>reachable</i>
+ * and an IPv4 type address; where the return value
+ * corresponds to a <i>loopback</i> address only if
+ * <code>true</code> is input for the <code>loopbackOk</code>
+ * parameter, and no non-loopback address satisfying
+ * the desired criteria can be found. If this method
+ * fails to find any address that satisfies the desired
+ * criteria, then <code>null</code> is returned.
+ *
+ * @throws SocketException if there is an error in the underlying
+ * I/O subsystem and/or protocol while retrieving the
+ * the network interfaces currently installed on the
+ * node.
+ *
+ * @throws IOException if a network error occurs while determining
+ * if a candidate return address is <i>reachable</i>.
+ */
+ public static String getDefaultIpv4Address(boolean loopbackOk)
+ throws SocketException, IOException
+ {
+ //get all nics on the current node
+ Enumeration<NetworkInterface> nics =
+ NetworkInterface.getNetworkInterfaces();
+ while( nics.hasMoreElements() ) {
+ NetworkInterface curNic = nics.nextElement();
+ List<InterfaceAddress> interfaceAddrs =
+ curNic.getInterfaceAddresses();
+ for(InterfaceAddress interfaceAddr : interfaceAddrs) {
+ InetAddress inetAddr = interfaceAddr.getAddress();
+ boolean isIpv4 = inetAddr instanceof Inet4Address;
+ boolean isLoopbackAddress = inetAddr.isLoopbackAddress();
+ if(isIpv4) {
+ if(isLoopbackAddress) continue;
+ boolean isReachable = inetAddr.isReachable(3*1000);
+ Inet4Address inet4Addr = (Inet4Address)inetAddr;
+ String retVal = inet4Addr.getHostAddress();
+
+ jiniConfigLogger.log
+ (CONFIG, "default IPv4 address: "+retVal);
+ utilLogger.log
+ (Level.TRACE, "default IPv4 address: "+retVal);
+ return retVal;
+ }
+ }
+ }
+
+ if(!loopbackOk) return null;
+
+ nics = NetworkInterface.getNetworkInterfaces();
+ while( nics.hasMoreElements() ) {
+ NetworkInterface curNic = nics.nextElement();
+ List<InterfaceAddress> interfaceAddrs =
+ curNic.getInterfaceAddresses();
+ for(InterfaceAddress interfaceAddr : interfaceAddrs) {
+ InetAddress inetAddr = interfaceAddr.getAddress();
+ boolean isIpv4 = inetAddr instanceof Inet4Address;
+ boolean isLoopbackAddress = inetAddr.isLoopbackAddress();
+ if(isIpv4) {
+ if(!isLoopbackAddress) continue;
+ boolean isReachable = inetAddr.isReachable(3*1000);
+ Inet4Address inet4Addr = (Inet4Address)inetAddr;
+ String retVal = inet4Addr.getHostAddress();
+
+ jiniConfigLogger.log
+ (CONFIG, "default IPv4 address: "+retVal);
+ utilLogger.log
+ (Level.TRACE, "default IPv4 address: "+retVal);
+ return retVal;
+ }
+ }
+ }
+ return null;
+ }
+
+ public static String getDefaultIpv4Address()
+ throws SocketException, IOException
+ {
+ return getDefaultIpv4Address(false);//localhost NOT ok
+ }
+
+ /**
* Intended for use by scripts.
*/
public static void main(String[] args) {
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/service/StressTestConcurrent.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/service/StressTestConcurrent.java 2010-07-23 11:04:00 UTC (rev 3268)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/service/StressTestConcurrent.java 2010-07-23 12:19:14 UTC (rev 3269)
@@ -791,10 +791,12 @@
throws IOException {
// explicitly set the log level for the load balancer.
- LoadBalancerService.log.setLevel(Level.INFO);
+//BTM LoadBalancerService.log.setLevel(Level.INFO);
+EmbeddedLoadBalancer.logger.setLevel(Level.INFO);
- final AbstractEmbeddedLoadBalancerService lbs = ((AbstractEmbeddedLoadBalancerService) ((EmbeddedFederation<?>) fed)
- .getLoadBalancerService());
+//BTM final AbstractEmbeddedLoadBalancerService lbs = ((AbstractEmbeddedLoadBalancerService) ((EmbeddedFederation<?>) fed)
+//BTM .getLoadBalancerService());
+final EmbeddedLoadBalancer lbs = ((EmbeddedLoadBalancer)((EmbeddedFederation)fed).getLoadBalancerService());
final ServiceScore[] fakeServiceScores = new ServiceScore[2];
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/ServiceImpl.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/ServiceImpl.java 2010-07-23 11:04:00 UTC (rev 3268)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/ServiceImpl.java 2010-07-23 12:19:14 UTC (rev 3269)
@@ -286,7 +286,7 @@
+", locators="
+Util.writeArrayElementsToString(locatorsToJoin));
- waitThread = new Util.WaitOnInterruptThread();
+ waitThread = new Util.WaitOnInterruptThread(logger);
waitThread.start();
readyState.ready();//ready to accept calls from clients
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/AbstractServicesManagerService.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/AbstractServicesManagerService.java 2010-07-23 11:04:00 UTC (rev 3268)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/AbstractServicesManagerService.java 2010-07-23 12:19:14 UTC (rev 3269)
@@ -67,6 +67,7 @@
* processes.
*/
+System.out.println("\nXXXXXXX AbstractServicesManagerService.add >>>>> KILLING process = "+service.name);
service.kill(true/* immediateShutdown */);
} catch (InterruptedException e) {
@@ -121,6 +122,7 @@
// halt our monitor tasks.
getFederation().cancelMonitoredTasks(true/* mayInterruptIfRunning */);
+System.out.println("\nXXXXXXX AbstractServicesManagerService.shutdown >>>>> KILLING CHILD PROCESSES");
// kill our children.
killChildProcesses(false/* immediateShutdown */);
@@ -146,6 +148,7 @@
// halt our monitor tasks.
getFederation().cancelMonitoredTasks(true/* mayInterruptIfRunning */);
+System.out.println("\nXXXXXXX AbstractServicesManagerService.shutdownNow >>>>> KILLING CHILD PROCESSES");
// kill our children.
killChildProcesses(true/* immediateShutdown */);
@@ -186,6 +189,7 @@
try {
+System.out.println("\nXXXXXXX AbstractServicesManagerService.killChildProcesses >>>>> calling Process.kill");
helper.kill(immediateShutdown);
} catch (Throwable t) {
@@ -301,6 +305,7 @@
/*
* Run startup.
*/
+System.out.println("\n*** AbstractServicesManagerService: setup() - ServicesManagerStartupTask().call() ***\n");
new ServicesManagerStartupTask(fed, config, true/* pushConfig */,
true/* restartServices */, this/* listener */).call();
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ManageLogicalServiceTask.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ManageLogicalServiceTask.java 2010-07-23 11:04:00 UTC (rev 3268)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ManageLogicalServiceTask.java 2010-07-23 12:19:14 UTC (rev 3269)
@@ -107,6 +107,7 @@
this.configZPath = configZPath;
this.children = children;
this.config = config;
+System.out.println("\nEEEE ManageLogicalServiceTask.constructor: configZPath="+this.configZPath);
}
@@ -178,6 +179,7 @@
*/
protected void newLogicalService() throws KeeperException, InterruptedException {
+System.out.println("\nEEEE ManageLogicalServiceTask.newLogicalService: className="+config.className);
if (INFO)
log.info("className=" + config.className);
@@ -194,31 +196,38 @@
/*
* Create zpath for the new logical service.
*/
+System.out.println("EEEE ManageLogicalServiceTask.newLogicalService: CREATING ZNODE ---> configZPath="+configZPath+", LOGICAL_SERVICE_PREFIX="+BigdataZooDefs.LOGICAL_SERVICE_PREFIX+", path="+(configZPath+"/"+BigdataZooDefs.LOGICAL_SERVICE_PREFIX));
final String logicalServiceZPath = zookeeper.create(configZPath + "/"
+ BigdataZooDefs.LOGICAL_SERVICE_PREFIX, new byte[0], acl,
CreateMode.PERSISTENT_SEQUENTIAL);
+System.out.println("EEEE ManageLogicalServiceTask.newLogicalService: CREATED ZNODE ---> logicalServiceZPath="+logicalServiceZPath);
/*
* The new znode (last path component of the new zpath).
*/
final String logicalServiceZNode = logicalServiceZPath
.substring(logicalServiceZPath.lastIndexOf('/') + 1);
+System.out.println("EEEE ManageLogicalServiceTask.newLogicalService: logicalServiceZNode="+logicalServiceZNode);
/*
* Create the znode that is the parent for the physical service
* instances (direct child of the logicalSevice znode).
*/
+System.out.println("EEEE ManageLogicalServiceTask.newLogicalService: CREATING ZNODE Physical Service Parent ---> logicalServiceZPath="+logicalServiceZPath+", PHYSICAL_SERVICES_CONTAINER="+BigdataZooDefs.PHYSICAL_SERVICES_CONTAINER+", path="+(logicalServiceZPath+"/"+BigdataZooDefs.PHYSICAL_SERVICES_CONTAINER));
zookeeper.create(logicalServiceZPath + "/"
+ BigdataZooDefs.PHYSICAL_SERVICES_CONTAINER, new byte[0], acl,
CreateMode.PERSISTENT);
+System.out.println("EEEE ManageLogicalServiceTask.newLogicalService: CREATED Parent ZNODE ---> "+(logicalServiceZPath+"/"+BigdataZooDefs.PHYSICAL_SERVICES_CONTAINER));
/*
* Create the znode for the election of the primary physical service for
* this logical service (direct child of the logicalSevice znode).
*/
+System.out.println("EEEE ManageLogicalServiceTask.newLogicalService: CREATING Election ZNODE ---> logicalServiceZPath="+logicalServiceZPath+", MASTER_ELECTION="+BigdataZooDefs.MASTER_ELECTION+", path="+(logicalServiceZPath+"/"+BigdataZooDefs.MASTER_ELECTION));
zookeeper.create(logicalServiceZPath + "/"
+ BigdataZooDefs.MASTER_ELECTION, new byte[0], acl,
CreateMode.PERSISTENT);
+System.out.println("EEEE ManageLogicalServiceTask.newLogicalService: CREATED Election ZNODE ---> "+(logicalServiceZPath+"/"+BigdataZooDefs.MASTER_ELECTION));
try {
@@ -239,14 +248,19 @@
* lock.
*/
+System.out.println("EEEE ManageLogicalServiceTask.newLogicalService: zroot="+(fed.getZooConfig().zroot)+", LOCKS_CREATE_PHYSICAL_SERVICE="+BigdataZooDefs.LOCKS_CREATE_PHYSICAL_SERVICE+", className="+config.className+", UNDERSCORE, logicalServiceZNode="+logicalServiceZNode);
+
final String lockNodeZPath = fed.getZooConfig().zroot + "/"
+ BigdataZooDefs.LOCKS_CREATE_PHYSICAL_SERVICE + "/"
+ config.className + "_" + logicalServiceZNode;
+System.out.println("EEEE ManageLogicalServiceTask.newLogicalService: *** lockNodeZPath="+lockNodeZPath);
+System.out.println("EEEE ManageLogicalServiceTask.newLogicalService: CREATING ZNODE for physical service host decision ---> logicalServiceZPath="+logicalServiceZPath);
zookeeper
.create(lockNodeZPath, SerializerUtil
.serialize(logicalServiceZPath), acl,
CreateMode.PERSISTENT);
+System.out.println("EEEE ManageLogicalServiceTask.newLogicalService: CREATED ZNODE for physical service host decision ---> logicalServiceZPath="+logicalServiceZPath);
if (INFO)
log.info("Created lock node: " + lockNodeZPath);
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/MonitorCreatePhysicalServiceLocksTask.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/MonitorCreatePhysicalServiceLocksTask.java 2010-07-23 11:04:00 UTC (rev 3268)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/MonitorCreatePhysicalServiceLocksTask.java 2010-07-23 12:19:14 UTC (rev 3269)
@@ -1,6 +1,7 @@
package com.bigdata.jini.start;
import java.io.IOException;
+import java.net.SocketException;
import java.net.UnknownHostException;
import java.nio.channels.FileLock;
import java.rmi.RemoteException;
@@ -121,12 +122,15 @@
* Note: This task runs until cancelled.
*/
public Void call() throws Exception {
+System.out.println("\nDDDD MonitorCreatePhysicalServiceLocksTask.call: zroot="+fed.getZooConfig().zroot);
+System.out.println("DDDD MonitorCreatePhysicalServiceLocksTask.call: BigdataZooDefs.LOCKS_CREATE_PHYSICAL_SERVICE="+BigdataZooDefs.LOCKS_CREATE_PHYSICAL_SERVICE);
/*
* All the locks of interest are direct children of this znode.
*/
final String locksZPath = fed.getZooConfig().zroot + "/"
+ BigdataZooDefs.LOCKS_CREATE_PHYSICAL_SERVICE;
+System.out.println("DDDD MonitorCreatePhysicalServiceLocksTask.call: locksZPath="+locksZPath);
while (true) {
@@ -170,15 +174,17 @@
protected void acquireWatcherAndRun(final String locksZPath)
throws KeeperException, InterruptedException {
-
final ZooKeeper zookeeper = fed.getZookeeper();
/*
* Note: The UnknownChildrenWatcher will keep trying until it is
* able to establish the watch.
*/
+System.out.println("DDDD MonitorCreatePhysicalServiceLocksTask.acquireWatcherAndRun: locksZPath="+locksZPath);
+System.out.println("DDDD MonitorCreatePhysicalServiceLocksTask.acquireWatcherAndRun: ADD EVENT WATCHER");
final UnknownChildrenWatcher watcher = new UnknownChildrenWatcher(
zookeeper, locksZPath);
+System.out.println("DDDD MonitorCreatePhysicalServiceLocksTask.acquireWatcherAndRun: EVENT WATCHER ADDED");
try {
@@ -188,7 +194,9 @@
try {
// child znode.
+System.out.println("DDDD MonitorCreatePhysicalServiceLocksTask.acquireWatcherAndRun: WAITING FOR EVENT ...");
final String znode = watcher.queue.take();
+System.out.println("DDDD MonitorCreatePhysicalServiceLocksTask.acquireWatcherAndRun: *** EVENT ---> znode="+znode);
if (znode.endsWith(ZLockImpl.INVALID)) {
@@ -204,6 +212,7 @@
// path to the new lock node.
final String lockNodeZPath = locksZPath + "/" + znode;
+System.out.println("DDDD MonitorCreatePhysicalServiceLocksTask.acquireWatcherAndRun: lockNodeZPath="+lockNodeZPath);
if (log.isInfoEnabled())
log.info("new lock: zpath=" + lockNodeZPath);
@@ -508,6 +517,8 @@
final String logicalServiceZPath = (String) SerializerUtil
.deserialize(zookeeper.getData(lockNodeZPath, false,
new Stat()));
+System.out.println("DDDD MonitorCreatePhysicalServiceLocksTask.checkConstraintsAndStartService: lockNodeZPath="+lockNodeZPath);
+System.out.println("DDDD MonitorCreatePhysicalServiceLocksTask.checkConstraintsAndStartService: **** logicalServiceZPath="+logicalServiceZPath);
/*
* If we hack off the last path component, we now have the zpath for the
@@ -515,6 +526,7 @@
*/
final String serviceConfigZPath = logicalServiceZPath.substring(0,
logicalServiceZPath.lastIndexOf('/'));
+System.out.println("DDDD MonitorCreatePhysicalServiceLocksTask.checkConstraintsAndStartService: **** serviceConfigZPath="+serviceConfigZPath);
if (log.isInfoEnabled())
log.info("logicalServiceZPath=" + logicalServiceZPath);
@@ -662,6 +674,7 @@
try {
if (!isLocalService(attributes)) {
+System.out.println("*** MonitorCreatePhysicalServiceLocks.restartIfNotRunning: NOT local service >>> RETURNING WITHOUT STARTING ["+serviceConfig.className+"]");
/*
* The service does not live on this host so we do not have
@@ -672,7 +685,9 @@
}
- } catch (UnknownHostException ex) {
+ } catch (IOException ex) {//SocketException or UnknownHostException
+System.out.println("*** MonitorCreatePhysicalServiceLocks.restartIfNotRunning: IOException ["+serviceConfig.className+"]");
+ex.printStackTrace();
log.warn("className=" + serviceConfig.className
+ ", physicalServiceZPath=" + physicalServiceZPath, ex);
@@ -686,6 +701,7 @@
+ ", physicalServiceZPath=" + physicalServiceZPath);
// block until we can evaluate this service for restart.
+System.out.println("*** MonitorCreatePhysicalServiceLocks.restartIfNotRunning: LOCAL SERVICE >>> start it ["+serviceConfig.className+"] ... BLOCKING");
lock.lockInterruptibly();
try {
@@ -695,6 +711,7 @@
if (!shouldRestartPhysicalService(zookeeper, serviceConfig,
physicalServiceZPath, attributes)) {
+System.out.println("*** MonitorCreatePhysicalServiceLocks.restartIfNotRunning: should NOT RESTART service ["+serviceConfig.className+"]");
if (log.isInfoEnabled())
log.info("Will not restart: className="
@@ -709,6 +726,8 @@
} catch (RemoteException ex) {
+System.out.println("*** MonitorCreatePhysicalServiceLocks.restartIfNotRunning: RemoteException ["+serviceConfig.className+"]");
+ex.printStackTrace();
log.error("RMI problem: className=" + serviceConfig.className
+ ", physicalServiceZPath=" + physicalServiceZPath, ex);
@@ -716,6 +735,8 @@
return false;
} catch (KeeperException ex) {
+System.out.println("*** MonitorCreatePhysicalServiceLocks.restartIfNotRunning: KeeperException ["+serviceConfig.className+"]");
+ex.printStackTrace();
log.error("Zookeeper problem: className="
+ serviceConfig.className + ", physicalServiceZPath="
@@ -740,6 +761,7 @@
final LinkedList<IServiceConstraint> violatedConstraints = new LinkedList<IServiceConstraint>();
if (!serviceConfig.canStartService(fed, violatedConstraints)) {
+System.out.println("*** MonitorCreatePhysicalServiceLocks.restartIfNotRunning: restart PREVENTED BY CONSTRAINTS ["+serviceConfig.className+"] >>> violatedConstraints="+violatedConstraints);
log.warn("Restart prevented by constraints: className="
+ serviceConfig.className + ", physicalServiceZPath="
+ physicalServiceZPath + ", violatedConstraints="
@@ -755,6 +777,7 @@
log.warn("Will restart: className=" + serviceConfig.className
+ ", physicalServiceZPath=" + physicalServiceZPath);
+System.out.println("*** MonitorCreatePhysicalServiceLocks.restartIfNotRunning: WILL RESTART service ["+serviceConfig.className+"] >>> restartPhysicalService");
return restartPhysicalService(serviceConfig,
logicalServiceZPath, physicalServiceZPath, attributes);
@@ -764,11 +787,15 @@
log.error("Service restart interrupted: className="
+ serviceConfig.className + ", physicalServiceZPath="
+ physicalServiceZPath);
+System.out.println("*** MonitorCreatePhysicalServiceLocks.restartIfNotRunning: InterruptedException ["+serviceConfig.className+"]");
+t.printStackTrace();
// rethrow so that the caller will see the interrupt.
throw t;
} catch (Throwable t) {
+System.out.println("*** MonitorCreatePhysicalServiceLocks.restartIfNotRunning: RESTART ERROR - "+t+" ["+serviceConfig.className+"]");
+t.printStackTrace();
// log and ignore.
log.error("Service restart error: className="
@@ -1148,10 +1175,11 @@
*
* @return <code>true</code> if the service lives on this host.
*
+ * @throws SocketException
* @throws UnknownHostException
*/
protected boolean isLocalService(final Entry[] attributes)
- throws UnknownHostException {
+ throws SocketException, UnknownHostException {
boolean isLocalHost = false;
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/RestartPersistentServices.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/RestartPersistentServices.java 2010-07-23 11:04:00 UTC (rev 3268)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/RestartPersistentServices.java 2010-07-23 12:19:14 UTC (rev 3269)
@@ -171,12 +171,15 @@
if (log.isInfoEnabled())
log.info("Considering " + serviceConfigZNodes.size()
+ " service configurations");
+System.out.println("\n*********************************************************");
+System.out.println("*** RestartPersistentServices.runOnce: Considering " + serviceConfigZNodes.size()+ " service configurations");
for (String serviceConfigZNode : serviceConfigZNodes) {
if (log.isInfoEnabled())
log.info("Considering service configuration: "
+ serviceConfigZNode);
+System.out.println("*** RestartPersistentServices.runOnce: Considering service configuration: "+ serviceConfigZNode);
/*
* Get the service configuration. We will need it iff we restart an
@@ -200,6 +203,7 @@
log.info("Considering " + logicalServiceZNodes.size()
+ " logical services configurations for "
+ serviceConfigZNode);
+System.out.println("*** RestartPersistentServices.runOnce: Considering " + logicalServiceZNodes.size()+ " logical services configurations for "+ serviceConfigZNode);
for (String logicalServiceZNode : logicalServiceZNodes) {
@@ -224,6 +228,7 @@
+ " physical services configurations for "
+ logicalServiceZNode + " of "
+ serviceConfigZNode);
+System.out.println("*** RestartPersistentServices.runOnce: Considering " + physicalServiceZNodes.size()+ " physical services configurations for "+ logicalServiceZNode + " of "+ serviceConfigZNode);
for (String physicalServiceZNode : physicalServiceZNodes) {
@@ -238,6 +243,7 @@
log.info("Considering: "
+ physicalServicesContainerZPath);
+System.out.println("*** RestartPersistentServices.runOnce >>> monitorCreatePhysicalServiceLocksTask.restartIfNotRunning <<<< [serviceConfig="+serviceConfig+", logicalServiceZPath="+logicalServiceZPath+", physicalServiceZPath="+physicalServiceZPath+"]");
monitorCreatePhysicalServiceLocksTask.restartIfNotRunning(
serviceConfig, logicalServiceZPath,
physicalServiceZPath, attributes);
@@ -247,6 +253,7 @@
}
}
+System.out.println("*********************************************************\n");
// Success.
return true;
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServiceConfigurationZNodeMonitorTask.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServiceConfigurationZNodeMonitorTask.java 2010-07-23 11:04:00 UTC (rev 3268)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServiceConfigurationZNodeMonitorTask.java 2010-07-23 12:19:14 UTC (rev 3269)
@@ -93,6 +93,13 @@
// zpath for the service configuration node.
serviceConfigZPath = zroot + "/" + BigdataZooDefs.CONFIG + "/" + className;
+
+System.out.println("\nGGGG ServiceConfigurationZNodeMonitorTask.constructor: zroot = "+zroot);
+System.out.println( "GGGG ServiceConfigurationZNodeMonitorTask.constructor: className = "+className);
+System.out.println( "GGGG ServiceConfigurationZNodeMonitorTask.constructor: LOCKS_SERVICE_CONFIG_MONITOR = "+BigdataZooDefs.LOCKS_SERVICE_CONFIG_MONITOR);
+System.out.println( "GGGG ServiceConfigurationZNodeMonitorTask.constructor: CONFIG = "+BigdataZooDefs.CONFIG);
+System.out.println( "GGGG ServiceConfigurationZNodeMonitorTask.constructor: lockZPath = "+lockZPath);
+System.out.println( "GGGG ServiceConfigurationZNodeMonitorTask.constructor: serviceConfigZPath = "+serviceConfigZPath);
}
@@ -216,6 +223,7 @@
* Note: This is done while we are holding the zlock. If we loose the
* zlock, the we cancel the watcher until we can reacquire the zlock.
*/
+System.out.println("GGGG ServiceConfigurationZNodeMonitorTask.runWithLock: SETTING WATCHER for ZNodes of interest ...");
final ServiceConfigurationHierarchyWatcher watcher = new ServiceConfigurationHierarchyWatcher(
zookeeper, serviceConfigZPath);
@@ -225,7 +233,11 @@
* them now and make sure that everything is in balance.
*/
final String[] watchedSet = watcher.getWatchedNodes();
-
+if(watchedSet == null) {
+ System.out.println("GGGG ServiceConfigurationZNodeMonitorTask.runWithLock: znodes-of-interest = "+watchedSet);
+}else{
+ System.out.println("GGGG ServiceConfigurationZNodeMonitorTask.runWithLock: znodes-of-interest = "+java.util.Arrays.asList(watchedSet));
+}
// Get the service configuration.
final ManagedServiceConfiguration config = (ManagedServiceConfiguration) SerializerUtil
.deserialize(zookeeper.getData(serviceConfigZPath, false,
@@ -237,16 +249,20 @@
while (true) {
+System.out.println("GGGG ServiceConfigurationZNodeMonitorTask.runWithLock: WAITING FOR EVENT ...");
final WatchedEvent e = watcher.queue.take();
if (!zlock.isLockHeld()) {
+System.out.println("GGGG ServiceConfigurationZNodeMonitorTask.runWithLock: LOST THE LOCK [event="+e+"]");
// Lost the lock.
break;
}
+System.out.println("GGGG ServiceConfigurationZNodeMonitorTask.runWithLock: HANDLE EVENT [event="+e+"]");
handleEvent(zookeeper, watcher, e);
+System.out.println("GGGG ServiceConfigurationZNodeMonitorTask.runWithLock: EVENT HANDLED [event="+e+"]");
}
@@ -307,6 +323,7 @@
if (config.serviceCount != children.size()) {
// adjust the #of logical service instances (blocks).
+System.out.println("GGGG ServiceConfigurationZNodeMonitorTask.runWithLock: *** NEW LOGICAL SERVICE TASK");
config.newLogicalServiceTask(fed, listener, serviceConfigZPath,
children).call();
@@ -339,6 +356,7 @@
throws KeeperException, InterruptedException {
for (String s : watchedSet) {
+System.out.println("GGGG ServiceConfigurationZNodeMonitorTask.balancePhysicalServices: serviceConfigZPath = "+serviceConfigZPath+", zpath="+s);
switch (ServiceConfigurationZNodeEnum
.getType(serviceConfigZPath, s)) {
@@ -348,10 +366,12 @@
// the parent is the logicalService.
final String logicalServiceZPath = s.substring(0, s
.lastIndexOf('/'));
+System.out.println("GGGG ServiceConfigurationZNodeMonitorTask.balancePhysicalServices: logicalServiceZPath = "+logicalServiceZPath);
// the znode of the logical service (last path component).
final String logicalServiceZNode = logicalServiceZPath
.substring(logicalServiceZPath.lastIndexOf('/') + 1);
+System.out.println("GGGG ServiceConfigurationZNodeMonitorTask.balancePhysicalServices: logicalServiceZNode = "+logicalServiceZNode);
// get children (the list of physical services).
final List<String> children = zookeeper.getChildren(
@@ -365,9 +385,11 @@
+ ", targetReplicationCount="
+ config.replicationCount + ", #children="
+ children.size() + ", children=" + children);
+System.out.println("GGGG ServiceConfigurationZNodeMonitorTask.balancePhysicalServices: serviceConfigZPath=" + serviceConfigZPath+ ", logicalServiceZPath=" + logicalServiceZPath+ ", targetReplicationCount="+ config.replicationCount + ", #children="+ children.size() + ", children=" + children);
// too few instances? : @todo handle too many instances also.
if (config.replicationCount > children.size()) {
+System.out.println("GGGG ServiceConfigurationZNodeMonitorTask.balancePhysicalServices: (config.replicationCount > children.size()) ["+config.replicationCount+" > "+children.size()+"]");
try {
@@ -384,9 +406,11 @@
+ "/" + config.className + "_"
+ logicalServiceZNode;
+System.out.println("GGGG ServiceConfigurationZNodeMonitorTask.balancePhysicalServices: *** CREATING logicalServiceZPath under lockNodeZPath ---> [zroot="+zroot+", LOCKS_CREATE_PHYSICAL_SERVICE="+BigdataZooDefs.LOCKS_CREATE_PHYSICAL_SERVICE+", className="+config.className+", UNDERSCORE, logicalServiceZNode="+logicalServiceZNode+"] ---> lockNodeZPath = "+lockNodeZPath);
zookeeper.create(lockNodeZPath, SerializerUtil
.serialize(logicalServiceZPath), fed
.getZooConfig().acl, CreateMode.PERSISTENT);
+System.out.println("GGGG ServiceConfigurationZNodeMonitorTask.balancePhysicalServices: *** CREATED ZNODE logicalServiceZNode under lockNodeZPath ---> logicalServiceZPath="+logicalServiceZPath+", lockNodeZPath="+lockNodeZPath);
if (log.isInfoEnabled())
log.info("Created lock node: " + lockNodeZPath);
@@ -396,7 +420,8 @@
// ignore.
}
-
+}else{
+System.out.println("GGGG ServiceConfigurationZNodeMonitorTask.balancePhysicalServices: (config.replicationCount NOT GREATER THAN children.size()) ["+config.replicationCount+" <= "+children.size()+"]");
}
default:
@@ -468,6 +493,7 @@
.getType(serviceConfigZPath, zpath)) {
case ServiceConfiguration: {
+System.out.println("GGGG ServiceConfigurationZNodeMonitorTask.handleEvent - CASE = ServiceConfiguration: HANDLE EVENT [event="+e+"]");
// Get the service configuration.
final ManagedServiceConfiguration config = (ManagedServiceConfiguration) SerializerUtil
@@ -481,6 +507,7 @@
}
case LogicalService: {
+System.out.println("GGGG ServiceConfigurationZNodeMonitorTask.handleEvent - CASE = LogicalService: HANDLE EVENT [event="+e+"]");
// Get the service configuration.
final ManagedServiceConfiguration config = (ManagedServiceConfiguration) SerializerUtil
@@ -494,12 +521,14 @@
}
case PhysicalServicesContainer: {
+System.out.println("GGGG ServiceConfigurationZNodeMonitorTask.handleEvent - CASE = PhysicalServiceContainter: HANDLE EVENT [event="+e+"]");
// Get the service configuration.
final ManagedServiceConfiguration config = (ManagedServiceConfiguration) SerializerUtil
.deserialize(zookeeper.getData(serviceConfigZPath, false,
new Stat()));
+System.out.println("GGGG ServiceConfigurationZNodeMonitorTask.handleEvent - CASE = PhysicalServiceContainter >>> balancePhysicalServices [event="+e+"]");
balancePhysicalServices(zookeeper, config, watchedSet);
break;
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServicesManagerServer.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServicesManagerServer.java 2010-07-23 11:04:00 UTC (rev 3268)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServicesManagerServer.java 2010-07-23 12:19:14 UTC (rev 3269)
@@ -58,7 +58,7 @@
import com.bigdata.service.DataService;
import com.bigdata.service.DefaultServiceFederationDelegate;
import com.bigdata.service.IDataService;
-import com.bigdata.service.ILoadBalancerService;
+//BTM import com.bigdata.service.ILoadBalancerService;
import com.bigdata.service.IMetadataService;
import com.bigdata.service.MetadataService;
import com.bigdata.service.jini.AbstractServer;
@@ -70,6 +70,9 @@
import com.sun.jini.start.ServiceDescriptor;
import com.sun.jini.start.ServiceStarter;
+//BTM
+import com.bigdata.service.LoadBalancer;
+
/**
* A class for bootstrapping a {@link JiniFederation} across a cluster based on
* a {@link Configuration} file describing some basic configuration parameters
@@ -215,7 +218,8 @@
* The two main services which require failover are the
* {@link ITransactionService} and the {@link IDataService} (which includes the
* {@link IMetadataService} as a special case). The loss of the
- * {@link ILoadBalancerService} is normally not critical as only history about
+BTM * {@link ILoadBalancerService} is normally not critical as only history about
+* {@link LoadBalancer} service is normally not critical as only history about
* the system load over time is lost.
*
* <h4>Transaction service</h4>
@@ -295,7 +299,8 @@
* and supports transactions over operations on the space. Gigaspaces has
* defined a variety of extensions that provide FIFO queues.
*
- * @todo The {@link MetadataService}, the {@link ILoadBalancerService}, and
+BTM * @todo The {@link MetadataService}, the {@link ILoadBalancerService}, and
+* @todo The {@link MetadataService}, the {@link LoadBalancer} service, and
* the {@link ITransactionService} MUST NOT have more than one logical
* instance in a federation. They can (eventually) have failover
* instances, but not peers. The {@link DataService} is the only one that
@@ -617,11 +622,42 @@
@Override
synchronized public void shutdownNow() {
+try {
+ net.jini.lookup.ServiceDiscoveryManager sdm = (server.getClient().getFederation()).getServiceDiscoveryManager();
+ net.jini.core.lookup.ServiceTemplate tmpl = new net.jini.core.lookup.ServiceTemplate(null, new Class[] {com.bigdata.service.Service.class}, null);
+ net.jini.core.lookup.ServiceItem[] serviceItems = sdm.lookup(tmpl, Integer.MAX_VALUE, Integer.MAX_VALUE, null, 5000L);
+ if(serviceItems.length > 0) {
+ for(int i=0; i<serviceItems.length; i++) {
+ Object srvcRef = serviceItems[i].service;
+ if(srvcRef == null) continue;
+ if(srvcRef instanceof net.jini.admin.Administrable) {
+ try {
+ Object srvcRefAdmin = ((net.jini.admin.Administrable)srvcRef).getAdmin();
+ if(srvcRefAdmin instanceof com.sun.jini.admin.DestroyAdmin) {
+ try {
+ ((com.sun.jini.admin.DestroyAdmin)srvcRefAdmin).destroy();
+ } catch(Throwable t) { }
+ } else {
+System.out.println("\nAAAA*** ServicesManagerService.shutdownNow: SERVICE ADMIN NOT INSTANCE OF DestroyAdmin ["+srvcRef+"]");
+ }//endif
+ } catch(Throwable t) {
+ t.printStackTrace();
+ }
+ } else {
+System.out.println("\nAAAA*** ServicesManagerService.shutdownNow: SERVICE NOT Administrable ["+srvcRef+"]");
+ }
+ }
+ }
+} catch(Throwable t) {
+ t.printStackTrace();
+}
+
// immediate service shutdown (blocks).
super.shutdownNow();
// jini service and server shutdown.
server.shutdownNow(false/*destroy*/);
+
}
@@ -678,6 +714,7 @@
final ConfigurationFile config = (ConfigurationFile) ConfigurationProvider
.getInstance(server.args);
+System.out.println("\nAAAA*** ServicesManagerService.AdministrableServicesManagerService: sighup() - fed.submitMonitoredTask(new ServicesManagerStartupTask() ***\n");
fed.submitMonitoredTask(new ServicesManagerStartupTask(fed,
config, pushConfig, restartServices, this));
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServicesManagerStartupTask.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServicesManagerStartupTask.java 2010-07-23 11:04:00 UTC (rev 3268)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServicesManagerStartupTask.java 2010-07-23 12:19:14 UTC (rev 3269)
@@ -139,6 +139,7 @@
for (ServiceConfiguration serviceConfig : serviceConfigurations) {
if (serviceConfig instanceof ZookeeperServerConfiguration) {
+System.out.println("\n---- ServicesManagerStartupTask.doStartup: startZookeeperService() ----");
startZookeeperService(config);
@@ -153,6 +154,7 @@
if (serviceConfig instanceof JiniCoreServicesConfiguration) {
+System.out.println("\n---- ServicesManagerStartupTask.doStartup: startJiniCoreServices() ----");
startJiniCoreServices(config);
}
@@ -169,10 +171,11 @@
final long begin = System.nanoTime();
- long nanos = TimeUnit.MILLISECONDS
- .toNanos(selfConfig.zookeeperDiscoveryTimeoutNanos);
-
+//BTM long nanos = TimeUnit.MILLISECONDS
+//BTm .toNanos(selfConfig.zookeeperDiscoveryTimeoutNanos);
+long nanos = selfConfig.zookeeperDiscoveryTimeoutNanos;
// await zookeeper connection.
+System.out.println("\n*** ServicesManagerStartupTask.doStartup: WAIT FOR ZOOKEEPER [zookeeperDiscoveryTimeoutNanos="+selfConfig.zookeeperDiscoveryTimeoutNanos+", nanos="+nanos+"] ***");
if (!fed.getZookeeperAccessor().awaitZookeeperConnected(nanos,
TimeUnit.NANOSECONDS)) {
@@ -189,6 +192,7 @@
* discovered and you can kill it if there is a problem.
*/
// await jini registrar(s)
+System.out.println("\n*** ServicesManagerStartupTask.doStartup: WAIT FOR LOOKUP ["+nanos+" nanoseconds] ***");
if (!fed.awaitJiniRegistrars(nanos, TimeUnit.NANOSECONDS)) {
throw new Exception(
@@ -197,6 +201,7 @@
}
}
+System.out.println("\n*** ServicesManagerStartupTask.doStartup: Lookup Service DISCOVERED *...
[truncated message content] |
|
From: <btm...@us...> - 2010-07-23 18:43:07
|
Revision: 3276
http://bigdata.svn.sourceforge.net/bigdata/?rev=3276&view=rev
Author: btmurphy
Date: 2010-07-23 18:43:00 +0000 (Fri, 23 Jul 2010)
Log Message:
-----------
merge -r3143::HEAD(3267) ~/bigdata/trunk ~/bigdata/branches/dev-btm [trunk --> branch dev-btm]
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/CollatorEnum.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/JDKSortKeyGenerator.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/GlobalRowStoreHelper.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestICUUnicodeKeyBuilder.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestJDKUnicodeKeyBuilder.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java
branches/dev-btm/bigdata/src/test/com/bigdata/cache/TestRingBuffer.java
branches/dev-btm/bigdata/src/test/com/bigdata/sparse/TestAll.java
branches/dev-btm/bigdata/src/test/com/bigdata/sparse/TestKeyEncodeDecode.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/MonitorCreatePhysicalServiceLocksTask.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/TruthMaintenance.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestNamedGraphs.java
branches/dev-btm/build.xml
branches/dev-btm/src/resources/config/bigdataCluster.config
branches/dev-btm/src/resources/config/bigdataCluster16.config
branches/dev-btm/src/resources/config/standalone/bigdataStandalone.config
Added Paths:
-----------
branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/AbstractUnicodeKeyBuilderTestCase.java
Removed Paths:
-------------
branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractUnicodeKeyBuilderTestCase.java
Property Changed:
----------------
branches/dev-btm/
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/
branches/dev-btm/dsi-utils/
branches/dev-btm/dsi-utils/src/test/
branches/dev-btm/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/src/resources/bin/config/
Property changes on: branches/dev-btm
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/bugfix-btm:2594-2779
/trunk:2575-2594,2596-2877,2882-2903,2910-3143
+ /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/bugfix-btm:2594-3237
/branches/fko:3150-3194
/trunk:2575-2594,2596-2877,2882-2903,2910-3269
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/CollatorEnum.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/CollatorEnum.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/CollatorEnum.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -10,7 +10,8 @@
/**
* The JDK bundles support for generating Unicode sort keys, but that
- * support does NOT include compressed sort keys.
+ * support does NOT include compressed sort keys and embeds <code>nul</code>
+ * bytes into its Unicode sort keys.
*/
JDK,
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -144,6 +144,7 @@
/**
* Representation includes all aspects of the {@link Serializable} state.
*/
+ @Override
public String toString() {
StringBuilder sb = new StringBuilder(getClass().getName());
@@ -270,14 +271,10 @@
/*
* Figure out which collator to use.
- *
- * Note: The default depends on whether or not the ICU library is on
- * the class path. When it is, we always default to the ICU library.
*/
collator = CollatorEnum.valueOf(getProperty(properties,
- Options.COLLATOR, (icu_avail ? CollatorEnum.ICU.toString()
- : CollatorEnum.JDK.toString())));
+ Options.COLLATOR, CollatorEnum.ICU.toString()));
// true iff the collator was _explicitly_ specified.
final boolean explicitCollatorChoice = getProperty(properties,
@@ -349,7 +346,7 @@
* Figure out the collator strength.
*/
- Object strength = null;
+ Object tmpStrength = null;
final String val = getProperty(properties, Options.STRENGTH);
@@ -357,24 +354,24 @@
try {
- strength = StrengthEnum.valueOf(val);
+ tmpStrength = StrengthEnum.valueOf(val);
} catch (RuntimeException ex) {
- strength = Integer.parseInt(val);
+ tmpStrength = Integer.parseInt(val);
}
}
if (log.isInfoEnabled())
- log.info(Options.STRENGTH + "=" + strength);
+ log.info(Options.STRENGTH + "=" + tmpStrength);
/*
* Note: MAY be null (when null, does not override the collator's
* default).
*/
- this.strength = strength;
+ this.strength = tmpStrength;
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -28,7 +28,6 @@
import java.util.Locale;
-import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import com.ibm.icu.text.Collator;
@@ -90,18 +89,6 @@
protected static final Logger log = Logger.getLogger(ICUSortKeyGenerator.class);
/**
- * True iff the {@link #log} level is INFO or less.
- */
- final protected boolean INFO = log.getEffectiveLevel().toInt() <= Level.INFO
- .toInt();
-
- /**
- * True iff the {@link #log} level is DEBUG or less.
- */
- final protected boolean DEBUG = log.getEffectiveLevel().toInt() <= Level.DEBUG
- .toInt();
-
- /**
* Used to encode unicode strings into compact byte[]s that have the same
* sort order (aka sort keys).
*/
@@ -128,7 +115,7 @@
this.locale = locale;
- if(INFO) log.info("locale="+locale);
+ if(log.isInfoEnabled()) log.info("locale="+locale);
this.collator = (RuleBasedCollator) Collator.getInstance(locale);
@@ -138,7 +125,7 @@
final int str = ((Integer) strength).intValue();
- if (INFO)
+ if (log.isInfoEnabled())
log.info("strength=" + str);
collator.setStrength(str);
@@ -147,7 +134,7 @@
StrengthEnum str = (StrengthEnum) strength;
- if (INFO)
+ if (log.isInfoEnabled())
log.info("strength=" + str);
switch (str) {
@@ -184,7 +171,7 @@
if (mode != null) {
- if(INFO) log.info("mode="+mode);
+ if(log.isInfoEnabled()) log.info("mode="+mode);
switch (mode) {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/JDKSortKeyGenerator.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/JDKSortKeyGenerator.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/JDKSortKeyGenerator.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -56,8 +56,8 @@
}
- public JDKSortKeyGenerator(Locale locale, Object strength,
- DecompositionEnum mode) {
+ public JDKSortKeyGenerator(final Locale locale, final Object strength,
+ final DecompositionEnum mode) {
if (locale == null)
throw new IllegalArgumentException();
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -180,9 +180,9 @@
* The buffer reference is used directly rather than making a
* copy of the data.
*/
- protected KeyBuilder(UnicodeSortKeyGenerator sortKeyGenerator, int len,
- byte[] buf) {
-
+ protected KeyBuilder(final UnicodeSortKeyGenerator sortKeyGenerator,
+ final int len, final byte[] buf) {
+
if (len < 0)
throw new IllegalArgumentException("len");
@@ -350,7 +350,7 @@
* The object responsible for generating sort keys from Unicode strings.
*
* The {@link UnicodeSortKeyGenerator} -or- <code>null</code> if Unicode
- * is not supported by this {@link KeyBuilder} instance.
+ * is not supported by this {@link IKeyBuilder} instance.
*/
final public UnicodeSortKeyGenerator getSortKeyGenerator() {
@@ -391,11 +391,11 @@
public KeyBuilder appendASCII(final String s) {
- int len = s.length();
+ int tmpLen = s.length();
- ensureFree(len);
+ ensureFree(tmpLen);
- for(int j=0; j<len; j++) {
+ for(int j=0; j<tmpLen; j++) {
char ch = s.charAt(j);
@@ -1356,6 +1356,11 @@
}
+ /**
+ * Create an instance for ASCII keys.
+ *
+ * @return The new instance.
+ */
public static IKeyBuilder newInstance() {
return newInstance(DEFAULT_INITIAL_CAPACITY);
@@ -1393,10 +1398,8 @@
/**
* Optional property specifies the library that will be used to generate
- * sort keys from Unicode data. The default always supports Unicode, but
- * the library choice depends on whether or not ICU library is found on
- * the classpath. When the ICU library is present, it is the default.
- * Otherwise the JDK library is the default. You may explicitly specify
+ * sort keys from Unicode data. The ICU library is the default.
+ * You may explicitly specify
* the library choice using one of the {@link CollatorEnum} values. The
* {@link CollatorEnum#ASCII} value may be used to disable Unicode
* support entirely, treating the characters as if they were ASCII. If
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -149,6 +149,7 @@
// allow client requests to finish normally.
new ShutdownHelper(threadPool, 10L/*logTimeout*/, TimeUnit.SECONDS) {
+ @Override
public void logTimeout() {
log.warn("Awaiting thread pool termination: elapsed="
@@ -170,6 +171,7 @@
new ShutdownHelper(scheduledExecutorService, 10L/* logTimeout */,
TimeUnit.SECONDS) {
+ @Override
public void logTimeout() {
log.warn("Awaiting sample service termination: elapsed="
@@ -650,12 +652,7 @@
TimeUnit.MILLISECONDS // unit
);
- addScheduledTask(//
- new StartDeferredTasksTask(),// task to run.
- 150, // initialDelay (ms)
- 2000, // delay
- TimeUnit.MILLISECONDS // unit
- );
+ getExecutorService().execute(new StartDeferredTasksTask());
// Setup locator.
resourceLocator = new DefaultResourceLocator(this,
@@ -888,12 +885,12 @@
*/
public boolean isServiceReady() {
- final AbstractClient<T> client = this.client;
+ final AbstractClient<T> thisClient = this.client;
- if (client == null)
+ if (thisClient == null)
return false;
- final IFederationDelegate<T> delegate = client.getDelegate();
+ final IFederationDelegate<T> delegate = thisClient.getDelegate();
if (delegate == null)
return false;
@@ -963,11 +960,11 @@
}
// @todo really, we should test like this everywhere.
- final AbstractClient client = this.client;
+ final AbstractClient thisClient = this.client;
- if (client != null && client.isConnected()) {
+ if (thisClient != null && thisClient.isConnected()) {
- client.getDelegate().serviceLeave(serviceUUID);
+ thisClient.getDelegate().serviceLeave(serviceUUID);
}
@@ -996,7 +993,7 @@
static private String ERR_SERVICE_NOT_READY = "Service is not ready yet.";
/**
- * This task runs periodically. Once {@link #getServiceUUID()} reports a
+ * This task runs once. Once {@link #getServiceUUID()} reports a
* non-<code>null</code> value, it will start an (optional)
* {@link AbstractStatisticsCollector}, an (optional) httpd service, and
* the (required) {@link ReportTask}.
@@ -1008,8 +1005,6 @@
* {@link LoadBalancer} service know which services exist, which is
* important for some of its functions.
* <p>
- * Once these task(s) have been started, this task will throw an exception
- * in order to prevent it from being re-executed.
*
* FIXME This should explicitly await jini registrar discovery, zookeeper
* client connected, and whatever other preconditions must be statisified
@@ -1032,8 +1027,7 @@
*/
final long begin = System.currentTimeMillis();
- public StartDeferredTasksTask() {
-
+ private StartDeferredTasksTask() {
}
/**
@@ -1043,8 +1037,6 @@
*/
public void run() {
- final boolean started;
-
try {
// /*
@@ -1061,9 +1053,7 @@
// return;
//
// }
-
- started = startDeferredTasks();
-
+ startDeferredTasks();
} catch (Throwable t) {
log.warn("Problem in report task?", t);
@@ -1072,60 +1062,56 @@
}
- if (started) {
-
- /*
- * Note: This exception is thrown once this task has executed
- * successfully.
- */
-
- throw new RuntimeException("Normal completion.");
-
- }
-
}
/**
* Starts performance counter collection once the service {@link UUID}
* is known.
*
- * @return <code>true</code> iff performance counter collection was
- * started.
- *
* @throws IOException
* if {@link IDataService#getServiceUUID()} throws this
* exception (it never should since it is a local method
* call).
*/
- protected boolean startDeferredTasks() throws IOException {
+ protected void startDeferredTasks() throws IOException {
// elapsed time since we started running this task.
final long elapsed = System.currentTimeMillis() - begin;
- if (getServiceUUID() == null) {
-
+ // Wait for the service ID to become available, trying every
+ // two seconds, while logging failures.
+ while (true) {
+ if (getServiceUUID() != null) {
+ break;
+ }
if (elapsed > 1000 * 10)
log.warn(ERR_NO_SERVICE_UUID + " : iface="
+ getServiceIface() + ", name=" + getServiceName()
+ ", elapsed=" + elapsed);
else if (log.isInfoEnabled())
- log.info(ERR_NO_SERVICE_UUID);
-
- return false;
-
+ log.info(ERR_NO_SERVICE_UUID);
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ }
}
- if (!isServiceReady()) {
-
+ // Wait for the service to become ready, trying every
+ // two seconds, while logging failures.
+ while (true) {
+ if (isServiceReady()) {
+ break;
+ }
if (elapsed > 1000 * 10)
log.warn(ERR_SERVICE_NOT_READY + " : iface="
+ getServiceIface() + ", name=" + getServiceName()
+ ", elapsed=" + elapsed);
else if (log.isInfoEnabled())
log.info(ERR_SERVICE_NOT_READY + " : " + elapsed);
-
- return false;
-
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ }
}
/*
@@ -1153,9 +1139,6 @@
// notify delegates that deferred startup has occurred.
AbstractFederation.this.didStart();
-
- return true;
-
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/sparse/GlobalRowStoreHelper.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/sparse/GlobalRowStoreHelper.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/sparse/GlobalRowStoreHelper.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -32,8 +32,12 @@
import org.apache.log4j.Logger;
+import com.bigdata.btree.DefaultTupleSerializer;
import com.bigdata.btree.IIndex;
import com.bigdata.btree.IndexMetadata;
+import com.bigdata.btree.keys.ASCIIKeyBuilderFactory;
+import com.bigdata.btree.keys.CollatorEnum;
+import com.bigdata.btree.keys.KeyBuilder;
import com.bigdata.journal.IIndexManager;
import com.bigdata.journal.ITx;
import com.bigdata.journal.TimestampUtility;
@@ -99,6 +103,25 @@
indexMetadata
.setSplitHandler(LogicalRowSplitHandler.INSTANCE);
+ if (CollatorEnum.JDK.toString().equals(
+ System.getProperty(KeyBuilder.Options.COLLATOR))) {
+ /*
+ * The JDK RulesBasedCollator embeds nul bytes in the
+ * Unicode sort keys. This makes them unsuitable for the
+ * SparseRowStore, which can not locate the start of the
+ * column name if there are embedded nuls in a Unicode
+ * primary key. As a work around, this forces an ASCII
+ * collation sequence if the JDK collator is the
+ * default. This is not ideal since non-ascii
+ * distinctions will be lost, but it is better than
+ * being unable to decode the column names.
+ */
+ log.warn("Forcing ASCII collator.");
+ indexMetadata
+ .setTupleSerializer(new DefaultTupleSerializer(
+ new ASCIIKeyBuilderFactory()));
+ }
+
// Register the index.
indexManager.registerIndex(indexMetadata);
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -49,17 +49,22 @@
* prefix.
* <p>
* The encoded schema name is followed by the {@link KeyType#getByteCode()} and
- * then by a <code>nul</code> byte. By searching for the <code>nul</code>
- * byte we can identify the end of the encoded schema name and also the data
- * type of the primary key. Most kinds of primary keys have a fixed length
- * encoding, e.g., {@link Long}, {@link Double}, etc. However, Unicode primary
- * keys have a variable length encoding which makes life more ... complex. Since
- * the keys need to reflect the total sort order we can not include the byte
- * count of the primary key in the key itself. The only reasonable approach is
- * to append a byte sequence to the key that never occurs within the generated
- * Unicode sort keys. We use a <code>nul</code> byte for this purpose since it
- * is not emitted by most Unicode collation implementations as it would cause
- * grief for C-language strings.
+ * then by a <code>nul</code> byte. By searching for the <code>nul</code> byte
+ * we can identify the end of the encoded schema name and also the data type of
+ * the primary key. Most kinds of primary keys have a fixed length encoding,
+ * e.g., {@link Long}, {@link Double}, etc.
+ * <p>
+ * Unicode primary keys have a variable length encoding which makes life more
+ * complex. For Unicode primary keys, we break with the collation order and use
+ * the UTF8 encoding of the key. This means that the primary key can be decoded
+ * and preserves hierarchical namespace clustering within the row store but does
+ * not impose a total sort order per Unicode sort key semantics. The only
+ * reasonable approach is to append a byte sequence to the key that never occurs
+ * within the generated Unicode sort keys. Again, we use a <code>nul</code> byte
+ * to mark the end of the Unicode primary key since it is not emitted by most
+ * Unicode collation implementations as it would cause grief for C-language
+ * strings. (However, see SparseRowStore.Options#PRIMARY_KEY_UNICODE_CLEAN} for
+ * information on backward compatibility.)
*
* @see Schema#fromKey(IKeyBuilder, Object)
* @see KeyType#getKeyType(byte)
@@ -166,15 +171,15 @@
* The decoded primary key.
*
* @throws UnsupportedOperationException
- * if the primary key can not be decoded (e.g., for
- * {@link KeyType#Unicode} keys).
+ * if the primary key can not be decoded.
*/
public Object getPrimaryKey() {
if(primaryKey == null) {
-
- throw new UnsupportedOperationException("Can not decode: keyType="+primaryKeyType);
-
+
+ throw new UnsupportedOperationException("Can not decode: keyType="
+ + primaryKeyType);
+
}
return primaryKey;
@@ -220,14 +225,13 @@
* Note: the KeyType byte occurs after the schema name bytes and before
* the [nul].
*/
+ int primaryKeyOffset = 0;
{
boolean found = false;
int schemaBytesLength = 0;
- int primaryKeyOffset = 0;
-
for (int i = 0; i < key.length; i++) {
if (key[i] == (byte) 0) {
@@ -256,7 +260,6 @@
this.primaryKeyTypeOffset = schemaBytesLength;
- this.primaryKeyOffset = primaryKeyOffset;
// note: ArrayIndexOutOfBounds with index==-1 means ICU library not on classpath!
this.primaryKeyType = KeyType.getKeyType(KeyBuilder.decodeByte(key[primaryKeyTypeOffset]));
@@ -273,42 +276,48 @@
primaryKeyLength = primaryKeyType.getEncodedLength();
+ this.primaryKeyOffset = primaryKeyOffset;
+
columnNameOffset = primaryKeyOffset + primaryKeyLength;
} else {
/*
- * Scan for the next [nul] byte.
+ * Scan for the next [nul] byte (ASCII).
*/
boolean found = false;
int primaryKeyLength = 0;
-
+
for (int i = primaryKeyOffset; i < key.length; i++) {
if (key[i] == (byte) 0) {
primaryKeyLength = i - primaryKeyOffset;
-
+
found = true;
-
+
break;
-
+
}
}
-
- if(!found) {
+ if (!found) {
+
throw new RuntimeException(
"Could not locate the end of the encoded schema name: keyType="
- + primaryKeyType+", key="+BytesUtil.toString(key));
+ + primaryKeyType + ", key="
+ + BytesUtil.toString(key));
}
this.primaryKeyLength = primaryKeyLength;
- // Note: also skips the [nul] byte terminating the primary key.
+ this.primaryKeyOffset = primaryKeyOffset;
+
+ // Note: also skips the [nul] byte terminating the primary
+ // key.
this.columnNameOffset = primaryKeyOffset + primaryKeyLength + 1;
}
@@ -327,10 +336,26 @@
primaryKey = KeyBuilder.decodeFloat(key, primaryKeyOffset);
break;
case Unicode:
- /*
- * Note: Decode is not possible for this case.
- */
- primaryKey = null;
+ if (SparseRowStore.primaryKeyUnicodeClean) {
+ final byte[] bytes = new byte[primaryKeyLength];
+ System.arraycopy(key, primaryKeyOffset, bytes, 0, primaryKeyLength);
+ try {
+ primaryKey = new String(bytes, SparseRowStore.UTF8);
+ } catch (UnsupportedEncodingException ex) {
+ throw new RuntimeException(
+ "Could not decode the primary key"
+ + ": primaryKeyOffset="
+ + primaryKeyOffset
+ + ", primaryKeyLength="
+ + primaryKeyLength + ", key="
+ + BytesUtil.toString(key));
+ }
+ } else {
+ /*
+ * Note: Decode is not possible for this case.
+ */
+ primaryKey = null;
+ }
break;
case ASCII:
primaryKey = KeyBuilder.decodeASCII(key, primaryKeyOffset,
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -154,7 +154,7 @@
/*
* Key builder stuff.
*/
-
+
/**
* Helper method appends a typed value to the compound key (this is used to
* get the primary key into the compound key).
@@ -172,8 +172,9 @@
*
* @see KeyDecoder
*/
- final protected IKeyBuilder appendPrimaryKey(IKeyBuilder keyBuilder, Object v, boolean successor) {
-
+ final protected IKeyBuilder appendPrimaryKey(final IKeyBuilder keyBuilder,
+ final Object v, final boolean successor) {
+
final KeyType keyType = getPrimaryKeyType();
if (successor) {
@@ -187,9 +188,27 @@
case Float:
return keyBuilder.append(successor(keyBuilder,((Number) v).floatValue()));
case Double:
- return keyBuilder.append(successor(keyBuilder,((Number) v).doubleValue()));
- case Unicode:
- return keyBuilder.appendText(v.toString(), true/*unicode*/, true/*successor*/).appendNul();
+ return keyBuilder.append(successor(keyBuilder, ((Number) v)
+ .doubleValue()));
+ case Unicode: {
+ final String tmp = v.toString();
+ if (SparseRowStore.primaryKeyUnicodeClean) {
+ try {
+ keyBuilder.append(
+ SuccessorUtil.successor(tmp
+ .getBytes(SparseRowStore.UTF8)))
+ .appendNul();
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ // primary key in backwards compatibility mode.
+ keyBuilder
+ .appendText(tmp, true/* unicode */, true/* successor */)
+ .appendNul();
+ }
+ return keyBuilder;
+ }
case ASCII:
return keyBuilder.appendText(v.toString(), false/*unicode*/, true/*successor*/).appendNul();
case Date:
@@ -212,8 +231,22 @@
return keyBuilder.append(((Number) v).floatValue());
case Double:
return keyBuilder.append(((Number) v).doubleValue());
- case Unicode:
- return keyBuilder.appendText(v.toString(),true/*unicode*/,false/*successor*/).appendNul();
+ case Unicode: {
+ final String tmp = v.toString();
+ if (SparseRowStore.primaryKeyUnicodeClean) {
+ try {
+ keyBuilder.append(tmp.getBytes(SparseRowStore.UTF8))
+ .appendNul();
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ // primary key in backwards compatibility mode.
+ keyBuilder.appendText(v.toString(), true/* unicode */,
+ false/* successor */).appendNul();
+ }
+ return keyBuilder;
+ }
case ASCII:
return keyBuilder.appendText(v.toString(),false/*unicode*/,false/*successor*/).appendNul();
case Date:
@@ -243,8 +276,8 @@
* {@link #toKey(Object)}, which correctly forms the successor
* key in all cases.
*/
- final private Object successor(IKeyBuilder keyBuilder,Object v) {
-
+ final private Object successor(final IKeyBuilder keyBuilder, final Object v) {
+
final KeyType keyType = getPrimaryKeyType();
switch(keyType) {
@@ -290,7 +323,8 @@
*
* @see KeyDecoder
*/
- final protected IKeyBuilder fromKey(IKeyBuilder keyBuilder,Object primaryKey) {
+ final protected IKeyBuilder fromKey(final IKeyBuilder keyBuilder,
+ final Object primaryKey) {
keyBuilder.reset();
@@ -319,8 +353,9 @@
*
* @return
*/
- final public byte[] getPrefix(IKeyBuilder keyBuilder,Object primaryKey) {
-
+ final public byte[] getPrefix(final IKeyBuilder keyBuilder,
+ final Object primaryKey) {
+
return fromKey(keyBuilder, primaryKey).getKey();
}
@@ -384,8 +419,8 @@
* @throws IllegalArgumentException
* if <i>col</i> is not valid as the name of a column.
*/
- public byte[] getKey(IKeyBuilder keyBuilder, Object primaryKey, String col,
- long timestamp) {
+ public byte[] getKey(final IKeyBuilder keyBuilder, final Object primaryKey,
+ final String col, final long timestamp) {
if (keyBuilder == null)
throw new IllegalArgumentException();
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -24,6 +24,7 @@
*/
package com.bigdata.sparse;
+import java.text.RuleBasedCollator;
import java.util.Iterator;
import java.util.Map;
@@ -33,7 +34,9 @@
import com.bigdata.btree.IIndex;
import com.bigdata.btree.IRangeQuery;
import com.bigdata.btree.ITuple;
+import com.bigdata.btree.IndexMetadata;
import com.bigdata.btree.filter.FilterConstructor;
+import com.bigdata.btree.keys.CollatorEnum;
import com.bigdata.btree.keys.IKeyBuilder;
import com.bigdata.journal.ITimestampService;
@@ -186,6 +189,17 @@
* md.setSplitHandler(LogicalRowSplitHandler.INSTANCE);
* </pre>
*
+ * Note: The JDK {@link RuleBasedCollator} embeds <code>nul</code> bytes in
+ * the Unicode sort keys. This makes them unsuitable for the row store which
+ * can not locate the start of the column name if there are embedded
+ * <code>nul</code>s in the primaryKey. Therefore, if you are using the
+ * {@link CollatorEnum#JDK} as your default collator, then you MUST override
+ * the {@link IndexMetadata} for the row store to use either an ASCII
+ * collator or the ICU collator. In general, the ICU collator is superior to
+ * the JDK collator and will be used by default. The ASCII collator is not
+ * ideal since non-ascii distinctions will be lost, but it is better than
+ * being unable to decode the data in the row store.
+ *
* @param ndx
* The index.
*/
@@ -1029,4 +1043,53 @@
}
+ /**
+ * Options for the {@link SparseRowStore}.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan
+ * Thompson</a>
+ * @version $Id$
+ */
+ public interface Options {
+
+ /**
+ * The primary key was originally written using a Unicode sort key.
+ * However, the JDK generates Unicode sort keys with embedded nuls and
+ * that broke the logic to detect the end of the Unicode primary keys.
+ * In order to accommodate this behavior, the Unicode primary key is now
+ * encoded as UTF8 which also has the advantage that we can decode
+ * Unicode primary keys. Standard prefix compression on the B+Tree
+ * should make up for the larger representation of the Unicode primary
+ * key in the B+Tree.
+ * <p>
+ * This change was introduced on 7/15/2010 in the trunk and breaks
+ * compatibility with earlier revisions of the {@link SparseRowStore}.
+ * This flag may be set to <code>false</code> for backward
+ * compatibility.
+ *
+ * @see Options#DEFAULT_PRIMARY_KEY_UNICODE_CLEAN
+ */
+ String PRIMARY_KEY_UNICODE_CLEAN = Schema.class.getName()
+ + ".primaryKey.unicodeClean";
+
+ /**
+ * FIXME Change over the [true] for the next release.
+ *
+ * @see https://sourceforge.net/apps/trac/bigdata/ticket/107
+ */
+ String DEFAULT_PRIMARY_KEY_UNICODE_CLEAN = "false";
+
+ }
+
+ /**
+ * This is a global option since it was always <code>false</code> for
+ * historical stores.
+ *
+ * @see Options#PRIMARY_KEY_UNICODE_CLEAN
+ */
+ final static transient boolean primaryKeyUnicodeClean = Boolean
+ .valueOf(System.getProperty(
+ SparseRowStore.Options.PRIMARY_KEY_UNICODE_CLEAN,
+ SparseRowStore.Options.DEFAULT_PRIMARY_KEY_UNICODE_CLEAN));
+
}
Deleted: branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractUnicodeKeyBuilderTestCase.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractUnicodeKeyBuilderTestCase.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractUnicodeKeyBuilderTestCase.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -1,183 +0,0 @@
-/*
-
-Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved.
-
-Contact:
- SYSTAP, LLC
- 4501 Tower Road
- Greensboro, NC 27410
- lic...@bi...
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
-*/
-/*
- * Created on Nov 29, 2007
- */
-
-package com.bigdata.btree;
-
-import java.util.Locale;
-import java.util.Properties;
-
-import junit.framework.TestCase2;
-
-import com.bigdata.btree.keys.DefaultKeyBuilderFactory;
-import com.bigdata.btree.keys.IKeyBuilder;
-import com.bigdata.btree.keys.KeyBuilder;
-import com.bigdata.btree.keys.StrengthEnum;
-import com.bigdata.btree.keys.KeyBuilder.Options;
-
-/**
- * Base class for the test suites that examine support for Unicode sort keys in
- * {@link KeyBuilder}.
- *
- * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
- */
-abstract public class AbstractUnicodeKeyBuilderTestCase extends TestCase2 {
-
- /**
- *
- */
- public AbstractUnicodeKeyBuilderTestCase() {
- }
-
- /**
- * @param arg0
- */
- public AbstractUnicodeKeyBuilderTestCase(String arg0) {
- super(arg0);
- }
-
- /**
- * Test ability to encode unicode data into a variable length byte[] that
- * allows direct byte-by-byte comparisons which maintain the local-specific
- * sort order of the original strings.
- */
- public void test_keyBuilder_unicode_string_key_us_primary() {
-
- /*
- * Setup an instance for US English with strength := PRIMARY.
- */
-
- Properties properties = getProperties();
-
- properties.setProperty(Options.USER_LANGUAGE, Locale.US.getLanguage());
-
- properties.setProperty(Options.USER_COUNTRY, Locale.US.getCountry());
-
- properties.setProperty(Options.STRENGTH, StrengthEnum.Primary
- .toString());
-
- final DefaultKeyBuilderFactory factory = new DefaultKeyBuilderFactory(
- properties);
-
- assertEquals(Locale.US.getLanguage(), factory.getLocale().getLanguage());
-
- assertEquals(Locale.US.getCountry(), factory.getLocale().getCountry());
-
- assertEquals(StrengthEnum.Primary, factory.getStrength());
-
- final IKeyBuilder keyBuilder = factory.getKeyBuilder();
-
-// // verify assumption under that configuration.
-// {
-// RuleBasedCollator usCollator = (RuleBasedCollator) Collator
-// .getInstance(Locale.US);
-//
-// usCollator.setStrength(Collator.PRIMARY);
-//
-// assertEquals(0, usCollator.compare("abc", "ABC"));
-// }
-
- byte[] key1 = keyBuilder.reset().append("abc").getKey();
- byte[] key2 = keyBuilder.reset().append("ABC").getKey();
- byte[] key3 = keyBuilder.reset().append("Abc").getKey();
-
- System.err.println("abc: "+BytesUtil.toString(key1));
- System.err.println("ABC: "+BytesUtil.toString(key2));
- System.err.println("Abc: "+BytesUtil.toString(key3));
-
- // all are equal using PRIMARY strength.
- assertEquals(0,BytesUtil.compareBytes(key1, key2));
- assertEquals(0,BytesUtil.compareBytes(key2, key3));
-
- }
-
- public void test_keyBuilder_unicode_string_key_us_identical() {
-
- /*
- * Setup an instance for US English with strength := IDENTICAL.
- */
-
- Properties properties = new Properties();
-
- properties.setProperty(Options.USER_LANGUAGE, Locale.US.getLanguage());
-
- properties.setProperty(Options.USER_COUNTRY, Locale.US.getCountry());
-
- properties.setProperty(Options.STRENGTH, ""+StrengthEnum.Identical);
-
- IKeyBuilder keyBuilder = KeyBuilder.newUnicodeInstance(properties);
-
-// // verify assumption under that configuration.
-// {
-// RuleBasedCollator usCollator = (RuleBasedCollator) Collator
-// .getInstance(Locale.US);
-//
-// usCollator.setStrength(Collator.IDENTICAL);
-//
-// assertNotSame(0, usCollator.compare("abc", "ABC"));
-// }
-
- // IKeyBuilder keyBuilder = new UnicodeKeyBuilder(usCollator,1000);
-
- byte[] key1 = keyBuilder.reset().append("abc").getKey();
- byte[] key2 = keyBuilder.reset().append("ABC").getKey();
- byte[] key3 = keyBuilder.reset().append("Abc").getKey();
-
- System.err.println("abc: "+BytesUtil.toString(key1));
- System.err.println("ABC: "+BytesUtil.toString(key2));
- System.err.println("Abc: "+BytesUtil.toString(key3));
-
- // verify ordering for IDENTICAL comparison.
- assertTrue(BytesUtil.compareBytes(key1, key2)<0);
- assertTrue(BytesUtil.compareBytes(key2, key3)>0);
-
- }
-
- /**
- * Test verifies that the trailing <code>nul</code> byte is not part of
- * the key when a unicode string is appended to an {@link IKeyBuilder}.
- * <p>
- * Note: The trailing <code>nul</code> byte is appended by the ICU library
- * in order to have compatibility with their C library, but it is not of
- * interest for Java processing. However, note that a <code>nul</code>
- * byte MAY be used to separate components of a complex key.
- */
- public void test_keyBuilder_unicode_String_noTrailingNul() {
-
- IKeyBuilder keyBuilder = KeyBuilder.newUnicodeInstance(getProperties());
-
- keyBuilder.append("Hello World!");
-
- byte[] key = keyBuilder.getKey();
-
- assertNotSame("Not expecting a trailing nul byte.", (byte) 0,
- key[key.length - 1]);
-
- }
-
-}
Copied: branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/AbstractUnicodeKeyBuilderTestCase.java (from rev 3269, trunk/bigdata/src/test/com/bigdata/btree/keys/AbstractUnicodeKeyBuilderTestCase.java)
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/AbstractUnicodeKeyBuilderTestCase.java (rev 0)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/AbstractUnicodeKeyBuilderTestCase.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -0,0 +1,342 @@
+/*
+
+Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+*/
+/*
+ * Created on Nov 29, 2007
+ */
+
+package com.bigdata.btree.keys;
+
+import java.util.Arrays;
+import java.util.Locale;
+import java.util.Properties;
+
+import junit.framework.TestCase2;
+
+import com.bigdata.btree.BytesUtil;
+import com.bigdata.btree.keys.DefaultKeyBuilderFactory;
+import com.bigdata.btree.keys.IKeyBuilder;
+import com.bigdata.btree.keys.KeyBuilder;
+import com.bigdata.btree.keys.StrengthEnum;
+import com.bigdata.btree.keys.KeyBuilder.Options;
+
+/**
+ * Base class for the test suites that examine support for Unicode sort keys in
+ * {@link KeyBuilder}.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+abstract public class AbstractUnicodeKeyBuilderTestCase extends TestCase2 {
+
+ /**
+ *
+ */
+ public AbstractUnicodeKeyBuilderTestCase() {
+ }
+
+ /**
+ * @param arg0
+ */
+ public AbstractUnicodeKeyBuilderTestCase(String arg0) {
+ super(arg0);
+ }
+
+ /**
+ * Test ability to encode unicode data into a variable length byte[] that
+ * allows direct byte-by-byte comparisons which maintain the local-specific
+ * sort order of the original strings.
+ */
+ public void test_keyBuilder_unicode_string_key_us_primary() {
+
+ /*
+ * Setup an instance for US English with strength := PRIMARY.
+ */
+
+ final Properties properties = getProperties();
+
+ properties.setProperty(Options.USER_LANGUAGE, Locale.US.getLanguage());
+
+ properties.setProperty(Options.USER_COUNTRY, Locale.US.getCountry());
+
+ properties.setProperty(Options.STRENGTH, StrengthEnum.Primary
+ .toString());
+
+ final DefaultKeyBuilderFactory factory = new DefaultKeyBuilderFactory(
+ properties);
+
+ assertEquals(Locale.US.getLanguage(), factory.getLocale().getLanguage());
+
+ assertEquals(Locale.US.getCountry(), factory.getLocale().getCountry());
+
+ assertEquals(StrengthEnum.Primary, factory.getStrength());
+
+ final IKeyBuilder keyBuilder = factory.getKeyBuilder();
+
+// // verify assumption under that configuration.
+// {
+// RuleBasedCollator usCollator = (RuleBasedCollator) Collator
+// .getInstance(Locale.US);
+//
+// usCollator.setStrength(Collator.PRIMARY);
+//
+// assertEquals(0, usCollator.compare("abc", "ABC"));
+// }
+
+ byte[] key1 = keyBuilder.reset().append("abc").getKey();
+ byte[] key2 = keyBuilder.reset().append("ABC").getKey();
+ byte[] key3 = keyBuilder.reset().append("Abc").getKey();
+
+ System.err.println("abc: "+BytesUtil.toString(key1));
+ System.err.println("ABC: "+BytesUtil.toString(key2));
+ System.err.println("Abc: "+BytesUtil.toString(key3));
+
+ // all are equal using PRIMARY strength.
+ assertEquals(0,BytesUtil.compareBytes(key1, key2));
+ assertEquals(0,BytesUtil.compareBytes(key2, key3));
+
+ }
+
+ public void test_keyBuilder_unicode_string_key_us_identical() {
+
+ /*
+ * Setup an instance for US English with strength := IDENTICAL.
+ */
+
+ final Properties properties = new Properties();
+
+ properties.setProperty(Options.USER_LANGUAGE, Locale.US.getLanguage());
+
+ properties.setProperty(Options.USER_COUNTRY, Locale.US.getCountry());
+
+ properties.setProperty(Options.STRENGTH, ""+StrengthEnum.Identical);
+
+ final IKeyBuilder keyBuilder = KeyBuilder.newUnicodeInstance(properties);
+
+// // verify assumption under that configuration.
+// {
+// RuleBasedCollator usCollator = (RuleBasedCollator) Collator
+// .getInstance(Locale.US);
+//
+// usCollator.setStrength(Collator.IDENTICAL);
+//
+// assertNotSame(0, usCollator.compare("abc", "ABC"));
+// }
+
+ // IKeyBuilder keyBuilder = new UnicodeKeyBuilder(usCollator,1000);
+
+ byte[] key1 = keyBuilder.reset().append("abc").getKey();
+ byte[] key2 = keyBuilder.reset().append("ABC").getKey();
+ byte[] key3 = keyBuilder.reset().append("Abc").getKey();
+
+ System.err.println("abc: "+BytesUtil.toString(key1));
+ System.err.println("ABC: "+BytesUtil.toString(key2));
+ System.err.println("Abc: "+BytesUtil.toString(key3));
+
+ // verify ordering for IDENTICAL comparison.
+ assertTrue(BytesUtil.compareBytes(key1, key2)<0);
+ assertTrue(BytesUtil.compareBytes(key2, key3)>0);
+
+ }
+
+ /**
+ * Test verifies that the trailing <code>nul</code> byte is not part of
+ * the key when a unicode string is appended to an {@link IKeyBuilder}.
+ * <p>
+ * Note: The trailing <code>nul</code> byte is appended by the ICU library
+ * in order to have compatibility with their C library, but it is not of
+ * interest for Java processing. However, note that a <code>nul</code>
+ * byte MAY be used to separate components of a complex key.
+ */
+ public void test_keyBuilder_unicode_String_noTrailingNul() {
+
+ final IKeyBuilder keyBuilder = KeyBuilder.newUnicodeInstance(getProperties());
+
+ keyBuilder.append("Hello World!");
+
+ final byte[] key = keyBuilder.getKey();
+
+ assertNotSame("Not expecting a trailing nul byte.", (byte) 0,
+ key[key.length - 1]);
+
+ }
+
+ /**
+ * Test verifies that the <code>nul</code> byte is not part of the key when
+ * a unicode string is appended to an {@link IKeyBuilder}.
+ * <p>
+ * Note: The {@link SparseRowStore} assumes that Unicode sort keys do not
+ * contain embedded <code>nul</code>s.
+ */
+ public void test_keyBuilder_unicode_String_noEmbeddedNuls() {
+
+ final IKeyBuilder keyBuilder = KeyBuilder
+ .newUnicodeInstance(getProperties());
+
+ keyBuilder.append("Hello World!");
+
+ final byte[] key = keyBuilder.getKey();
+
+ for (int i = 0; i < key.length; i++) {
+ if (key[i] == 0)
+ fail("Embedded nuls: key=" + BytesUtil.toString(key));
+ }
+
+ }
+
+ /**
+ * Test of the ability to normalize trailing pad characters.
+ */
+ public void test_keyBuilder_normalizeTrailingPadCharacters() {
+
+ final KeyBuilder keyBuilder = (KeyBuilder) KeyBuilder
+ .newUnicodeInstance(getProperties());
+
+ assertEquals(//
+ keyBuilder.normalizeText(""),//
+ keyBuilder.normalizeText(" ")//
+ );
+ assertEquals(//
+ keyBuilder.normalizeText(""),//
+ keyBuilder.normalizeText(" ")//
+ );
+ assertEquals(//
+ keyBuilder.normalizeText(""),//
+ keyBuilder.normalizeText(" ")//
+ );
+ assertEquals(//
+ keyBuilder.normalizeText(" "),//
+ keyBuilder.normalizeText(" ")//
+ );
+ assertEquals(//
+ keyBuilder.normalizeText("abc"),//
+ keyBuilder.normalizeText("abc ")//
+ );
+ assertEquals(//
+ keyBuilder.normalizeText(" abc"),//
+ keyBuilder.normalizeText(" abc ")//
+ );
+ assertNotSame(//
+ keyBuilder.normalizeText("abc"),//
+ keyBuilder.normalizeText(" abc ")//
+ );
+
+ }
+
+ /**
+ * Test verifies that very long strings are truncated.
+ *
+ * @todo verify that trailing whitespace is removed after truncation rather
+ * than before truncation.
+ */
+ public void test_keyBuilder_normalizeTruncatesVeryLongStrings() {
+
+ final KeyBuilder keyBuilder = (KeyBuilder)KeyBuilder.newUnicodeInstance(getProperties());
+
+ final String text = TestKeyBuilder.getMaximumLengthText();
+
+ assertEquals(//
+ keyBuilder.normalizeText(text),//
+ keyBuilder.normalizeText(text+"abc")//
+ );
+
+ }
+
+ /**
+ * Test verifies the order among unicode sort keys, including verifying that
+ * the pad byte causes a prefix such as "bro" to sort before a term which
+ * extends that prefix, such as "brown".
+ */
+ public void test_keyBuilder_unicode_order() {
+
+ final KeyBuilder keyBuilder = (KeyBuilder) KeyBuilder.newUnicodeInstance(getProperties());
+
+ final KVO<String>[] a = new KVO[] {
+
+ new KVO<String>(keyBuilder.asSortKey("bro"),null,"bro"),
+ new KVO<String>(keyBuilder.asSortKey("brown"),null,"brown"),
+ new KVO<String>(keyBuilder.asSortKey("bre"),null,"bre"),
+ new KVO<String>(keyBuilder.asSortKey("break"),null,"break"),
+
+ };
+
+ // sort by the assigned sort keys.
+ Arrays.sort(a);
+
+ /*
+ * verify that "bre(ak)" is before "bro(wn)" and that "bre" is before
+ * "break" and "bro" is before "brown".
+ */
+ assertEquals("bre", a[0].obj);
+ assertEquals("break", a[1].obj);
+ assertEquals("bro", a[2].obj);
+ assertEquals("brown", a[3].obj);
+
+ }
+
+ /**
+ * <p>
+ * Test that lexiographic order is maintain when a variable length Unicode
+ * field is followed by another field. This test works by comparing the
+ * original multi-field key with the multi-field key formed from the
+ * successor of the Unicode field followed by the other field:
+ * </p>
+ *
+ * <pre>
+ *
+ * [text][nextValue] LT [successor(text)][nextValue]
+ *
+ * </pre>
+ */
+ public void test_keyBuilder_multiField_unicode() {
+
+ final KeyBuilder keyBuilder = (KeyBuilder) KeyBuilder
+ .newUnicodeInstance(getProperties());
+
+ TestKeyBuilder.doMultiFieldTests(true/* unicode */, keyBuilder);
+
+ /*
+ * Now test some strings that contain code points outside of the 8-bit
+ * range.
+ */
+
+// final KeyBuilder keyBuilder = (KeyBuilder) KeyBuilder
+// .newUnicodeInstance();
+
+ final boolean unicode = true;
+ {
+
+ // Note: This is "Japanese" in kanji.
+ String text = "\u65E5\u672C\u8A9E / \u306B\u307B\u3093\u3054";
+
+ TestKeyBuilder.doMultiFieldTest(keyBuilder, unicode, text, (byte) 0);
+ TestKeyBuilder.doMultiFieldTest(keyBuilder, unicode, text, (byte) 1);
+ TestKeyBuilder.doMultiFieldTest(keyBuilder, unicode, text, (byte) -1);
+ TestKeyBuilder.doMultiFieldTest(keyBuilder, unicode, text, Byte.MIN_VALUE);
+ TestKeyBuilder.doMultiFieldTest(keyBuilder, unicode, text, Byte.MAX_VALUE);
+ }
+
+ }
+
+}
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestICUUnicodeKeyBuilder.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestICUUnicodeKeyBuilder.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestICUUnicodeKeyBuilder.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -31,7 +31,6 @@
import java.util.Locale;
import java.util.Properties;
-import com.bigdata.btree.AbstractUnicodeKeyBuilderTestCase;
import com.bigdata.btree.BytesUtil;
import com.bigdata.btree.keys.KeyBuilder.Options;
import com.ibm.icu.text.Collator;
@@ -39,15 +38,16 @@
/**
* Tests for Unicode support in {@link KeyBuilder}.
*
- * @todo test w/ and w/o the ICU integration (can be choosen at run time). Note
- * that some tests are specific to the ICU libraries at this time.
+ * @todo test w/ and w/o the ICU integration (can be chosen at run time via
+ * concrete subclasses). Note that some tests are specific to the ICU
+ * libraries at this time.
*
* @todo write performance test for encoding strings, possibly in the context of
* parsed rdf data, and see if there are any easy wins in how the encoding
* to a sort key is handled or in alignment of the apis.
*
* @todo compare performance of the ICU and JDK libraries in some application
- * contexts. compare performance of the JNI ICU library as well.
+ * contexts. compare performance of the JNI ICU library as well.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
@@ -69,7 +69,7 @@
public Properties getProperties() {
- Properties properties = new Properties(super.getProperties());
+ final Properties properties = new Properties(super.getProperties());
properties.setProperty(Options.COLLATOR,CollatorEnum.ICU.toString());
@@ -79,11 +79,12 @@
public void test_correctCollator() {
- Properties properties = getProperties();
-
- log.info("properties="+properties);
-
- KeyBuilder keyBuilder = (KeyBuilder) KeyBuilder
+ final Properties properties = getProperties();
+
+ if (log.isInfoEnabled())
+ log.info("properties=" + properties);
+
+ final KeyBuilder keyBuilder = (KeyBuilder) KeyBuilder
.newUnicodeInstance(properties);
assertEquals(ICUSortKeyGenerator.class, keyBuilder
@@ -109,13 +110,13 @@
* Setup for US English.
*/
- Properties properties = new Properties();
+ final Properties properties = new Properties();
properties.setProperty(Options.USER_LANGUAGE, Locale.US.getLanguage());
properties.setProperty(Options.USER_COUNTRY, Locale.US.getCountry());
- int[] strengths = new int[] {
+ final int[] strengths = new int[] {
Collator.PRIMARY,
Collator.SECONDARY,
Collator.TERTIARY,
@@ -174,7 +175,7 @@
* @param collator
* The collator.
*
- * @return True iff the collector differenties between the string and its
+ * @return True iff the collector differentiates between the string and its
* successor (formed by appending a nul character) in its generated
* sort keys.
*/
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestJDKUnicodeKeyBuilder.java
===========...
[truncated message content] |
|
From: <btm...@us...> - 2010-07-28 20:48:02
|
Revision: 3337
http://bigdata.svn.sourceforge.net/bigdata/?rev=3337&view=rev
Author: btmurphy
Date: 2010-07-28 20:47:53 +0000 (Wed, 28 Jul 2010)
Log Message:
-----------
[branch dev-btm]: trac #126 - Uses of InetAddress.getLocalAddress should be changed to a mechanism that returns a non-loopback address
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/MoveTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractEmbeddedLoadBalancerService.java
branches/dev-btm/bigdata/src/java/com/bigdata/util/config/NicUtil.java
branches/dev-btm/bigdata/src/test/com/bigdata/counters/httpd/TestCounterSetHTTPDServer.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestResourceService.java
branches/dev-btm/bigdata/src/test/com/bigdata/test/ExperimentDriver.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/config/executor.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/JiniServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/MaxClientServicesPerHostConstraint.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/MaxDataServicesPerHostConstraint.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ZookeeperServerConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/JiniCoreServicesProcessHelper.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/ZookeeperProcessHelper.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/config/loadbalancer.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/shardlocator.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/process/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/process/config/process.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/config/quorum.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/LoadBalancerServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/LookupStarter.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/config/lookup.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/config/shard.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/config/transaction.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/ConfigDeployUtil.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/default-deploy.properties
branches/dev-btm/bigdata-jini/src/java/com/bigdata/zookeeper/ZooHelper.java
branches/dev-btm/bigdata-jini/src/resources/config/bigdataStandaloneTesting.config
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/config/TestZookeeperServerEntry.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/testReggie.config
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/testStartJini.config
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/AbstractServerTestCase.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/zookeeper/testzoo.config
branches/dev-btm/bigdata-lubm/src/java/edu/lehigh/swat/bench/ubt/Test.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/metrics/TestMetrics.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/AbstractServerTestCase.java
branches/dev-btm/build.properties
branches/dev-btm/build.xml
branches/dev-btm/src/resources/bin/config/browser.config
branches/dev-btm/src/resources/bin/pstart
branches/dev-btm/src/resources/config/bigdataCluster.config
branches/dev-btm/src/resources/config/jini/reggie.config
branches/dev-btm/src/resources/scripts/bigdataup
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -48,6 +48,8 @@
import com.bigdata.counters.win.StatisticsCollectorForWindows;
import com.bigdata.io.DirectBufferPool;
import com.bigdata.rawstore.Bytes;
+import com.bigdata.util.config.ConfigDeployUtil;
+import com.bigdata.util.config.NicUtil;
import com.bigdata.util.httpd.AbstractHTTPD;
/**
@@ -64,7 +66,6 @@
* and Un*x platforms so as to support the declared counters on all platforms.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
abstract public class AbstractStatisticsCollector implements IStatisticsCollector {
@@ -83,32 +84,10 @@
String s;
try {
-
-// hostname = InetAddress.getLocalHost().getHostName();
-
- s = InetAddress.getLocalHost().getCanonicalHostName();
-
- } catch (UnknownHostException e) {
-
- try {
-
- s = InetAddress.getLocalHost().getHostName();
-
- } catch(UnknownHostException e2) {
-
- final String msg = "Could not resolve hostname";
- try {
- log.error(msg);
- } catch(Throwable t) {
- System.err.println(msg);
- }
-
- s = "localhost";
-
- //throw new AssertionError(e);
-
- }
-
+ s = NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
+ } catch(Throwable t) {//for now, maintain same failure logic as used previously
+ t.printStackTrace();
+ s = NicUtil.getIpAddressByLocalHost();
}
fullyQualifiedHostName = s;
@@ -520,7 +499,6 @@
* Options for {@link AbstractStatisticsCollector}
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public interface Options {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/resources/MoveTask.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/resources/MoveTask.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/resources/MoveTask.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -63,6 +63,8 @@
import com.bigdata.service.IMetadataService;
import com.bigdata.service.MetadataService;
import com.bigdata.service.ResourceService;
+import com.bigdata.util.config.ConfigDeployUtil;
+import com.bigdata.util.config.NicUtil;
/**
* Task moves an index partition to another {@link IDataService}.
@@ -155,7 +157,6 @@
* need to do the atomic update phase.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public class MoveTask extends AbstractPrepareTask<MoveResult> {
@@ -421,7 +422,6 @@
* successful.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*
* @todo optimization to NOT send an empty index segment if there are no
* buffered writes on the live journal.
@@ -434,6 +434,7 @@
private final UUID targetDataServiceUUID;
private final int targetIndexPartitionId;
private final Event parentEvent;
+ private final InetAddress thisInetAddr;
/**
*
@@ -481,6 +482,11 @@
this.targetIndexPartitionId = targetIndexPartitionId;
this.parentEvent = parentEvent;
+ try {
+ this.thisInetAddr = InetAddress.getByName(NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.dataNetwork"), false));
+ } catch(Throwable t) {
+ throw new IllegalArgumentException(t.getMessage(), t);
+ }
}
/**
@@ -591,7 +597,7 @@
targetIndexPartitionId,//
historicalWritesBuildResult.segmentMetadata,//
bufferedWritesBuildResult.segmentMetadata,//
- InetAddress.getLocalHost(),//
+ thisInetAddr,
resourceManager
.getResourceServicePort()//
)).get();
@@ -856,7 +862,6 @@
* until the "receive" operation is complete.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
private static class IsIndexRegistered_UsingWriteService implements
IIndexProcedure {
@@ -896,7 +901,6 @@
* @see InnerReceiveIndexPartitionTask
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
protected static class ReceiveIndexPartitionTask extends DataServiceCallable<Void> {
@@ -1039,7 +1043,6 @@
* source data service.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
private static class InnerReceiveIndexPartitionTask extends AbstractTask<Void> {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractEmbeddedLoadBalancerService.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractEmbeddedLoadBalancerService.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractEmbeddedLoadBalancerService.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -5,6 +5,9 @@
import java.util.Properties;
import java.util.UUID;
+import com.bigdata.util.config.ConfigDeployUtil;
+import com.bigdata.util.config.NicUtil;
+
/**
* Embedded {@link LoadBalancerService}.
*
@@ -14,7 +17,7 @@
abstract public class AbstractEmbeddedLoadBalancerService extends LoadBalancerService {
// final private UUID serviceUUID;
- final private String hostname;
+ private String hostname = NicUtil.getIpAddressByLocalHost();//for now, maintain the same failure logic as in constructor
public AbstractEmbeddedLoadBalancerService(UUID serviceUUID,
Properties properties) {
@@ -28,17 +31,11 @@
setServiceUUID(serviceUUID);
- String hostname;
try {
-
- hostname = Inet4Address.getLocalHost().getCanonicalHostName();
-
- } catch (UnknownHostException e) {
-
- hostname = "localhost";
-
+ this.hostname = NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
+ } catch(Throwable t) {
+ t.printStackTrace();
}
- this.hostname = hostname;
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/util/config/NicUtil.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/util/config/NicUtil.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/util/config/NicUtil.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -507,54 +507,42 @@
/**
* Special-purpose convenience method that returns a
* <code>String</code> value representing the ip address of
- * the current node; where the value that is returned is
- * determined according to following criteria:
+ * the current node.
* <p>
- * <ul>
- * <li> If a non-<code>null</code> value is input for the
- * <code>systemPropertyName</code> parameter, then
- * this is viewed as a declaration by the caller that
- * that the system property with that given value
- * should take precedence over all other means of
- * determining the desired ip address. As such, this
- * method determines if a system property having the
- * given has indeed been set and, if it has, returns
- * the ip address of the nic having that name; or
- * <code>null</code> if there is no nic with the
- * desired name installed on the node.
- * <li> If a non-<code>null</code> value is input for the
- * <code>systemPropertyName</code> parameter, but
- * no system property with that name has been set,
- * and <code>true</code> has been passed in for
- * the <code>fallbackOk</code> parameter, then this
- * method will return the IPV4 based address of the
- * first reachable nic that is found on the node.
- * Upon failing to find such an address, if the
- * <code>loopbackOk</code> parameter is also
- * <code>true</code>, then this method will return
- * the <i>loop back</i> address of the node; otherwise
- * <code>null</code> is returned.
- * <li> If <code>null</code> is input for the
- * <code>systemPropertyName</code> parameter, but
- * a non-<code>null</code> value is input for the
- * <code>defaultNic</code> parameter, then this
- * method returns the ip address of the nic having
- * that name; or <code>null</code> if there is no
- * nic with the desired default name installed on the
- * node.
- * <li> If <code>null</code> is input for both the
- * <code>systemPropertyName</code> parameter and the
- * <code>defaultNic</code> parameter, and if the
- * <code>fallbackOk</code> parameter is <code>true</code>,
- * then this method will return the IPV4 based address
- * of the first reachable nic that is found on the node.
- * Upon failing to find such an address, if the
- * <code>loopbackOk</code> parameter is also
- * <code>true</code>, then this method will return
- * the <i>loop back</i> address of the node; otherwise
- * <code>null</code> is returned.
- * </ul>
+ * If a non-<code>null</code> value is input for the
+ * <code>systemPropertyName</code> parameter, then this
+ * method first determines if a system property with
+ * name equivalent to the given value has been set and,
+ * if it has, returns the ip address of the nic whose name
+ * is equivalent to that system property value; or
+ * <code>null</code> if there is no nic with the desired
+ * name installed on the node.
* <p>
+ * If there is no system property whose name is the value
+ * of the <code>systemPropertyName</code> parameter, and
+ * if the value "default" is input for the
+ * <code>defaultNic</code> parameter, then this method
+ * will return the IPV4 based address of the first reachable
+ * nic that can be found on the node; otherwise, if a
+ * non-<code>null</code> value not equal to "default" is
+ * input for the the <code>defaultNic</code> parameter,
+ * then this method returns the ip address of the nic
+ * corresponding to that given name; or <code>null</code>
+ * if there is no such nic name installed on the node.
+ * <p>
+ * If, on the other hand, <code>null</code> is input for
+ * the <code>systemPropertyName</code> parameter, then
+ * this method will attempt to find the desired ip address
+ * using only the value of the <code>defaultNic</code>,
+ * and applying the same search criteria as described
+ * above.
+ * <p>
+ * Note that in all cases, if <code>true</code> is input
+ * for the <code>loopOk</code> parameter, then upon failing
+ * to find a valid ip address using the specified search
+ * mechanism, this method will return the <i>loop back</i>
+ * address; otherwise, <code>null</code> is returned.
+ * <p>
* This method can be called from within a configuration
* as well as from within program control.
*
@@ -568,33 +556,24 @@
* the name of the network interface
* whose ip address should be returned
* if <code>null</code> is input for the
+ * <code>systemPropertyName</code> parameter,
+ * or if there is no system property with
+ * name equivalent the value of the
* <code>systemPropertyName</code> parameter.
*
- * @param fallbackOk if <code>true</code>, then if either
- * no system property is set having the
- * name referenced by the
- * <code>systemPropertyName</code> parameter,
- * or if <code>null</code> is input for both
- * the <code>systemPropertyName</code>
- * parameter and the <code>defaultNic</code>
- * parameter, return the IPV4 based address
- * of the first reachable network interface
- * that can be found on the node.
+ * @param loopbackOk if <code>true</code>, then return the
+ * <i>loop back</i> address upon failure
+ * to find a valid ip address using the
+ * search criteria specified through the
+ * <code>systemPropertyName</code> and
+ * <code>defaultNic</code> parameters.
*
- * @param loopbackOk if <code>true</code>, and if <code>true</code>
- * is also input for the <code>fallbackOk</code>
- * parameter, then if this method attempts,
- * but fails, to find a valid IPV4 fallback
- * address, then the node's <i>loop back</i>
- * address is returned.
- *
* @return a <code>String</code> representing an ip address associated
* with the current node; where the value that is returned is
* determined according to the criteria described above.
*/
public static String getIpAddress(String systemPropertyName,
String defaultNic,
- boolean fallbackOk,
boolean loopbackOk)
throws SocketException, IOException
{
@@ -613,32 +592,30 @@
}
if(propSet) {
return getIpAddress(nicName, 0, loopbackOk);
- } else {//desired system property not set, try fallback
- if(fallbackOk) {
- return getDefaultIpv4Address(loopbackOk);
+ } else {//system property not set, try default and/or fallback
+ if(defaultNic != null) {
+ if( defaultNic.equals("default") ) {
+ return getDefaultIpv4Address(loopbackOk);
+ } else {
+ return getIpAddress(defaultNic, 0, loopbackOk);
+ }
} else {
return null;
}
}
- } else {//no system property name provided, try default nic
+ } else {//no system property name provided, try default
if(defaultNic != null) {
- return getIpAddress(defaultNic, 0, loopbackOk);
- } else {//no default nic provided, try fallback
- if(fallbackOk) {
+ if( defaultNic.equals("default") ) {
return getDefaultIpv4Address(loopbackOk);
} else {
- return null;
+ return getIpAddress(defaultNic, 0, loopbackOk);
}
+ } else {
+ return getIpAddress(null, loopbackOk);
}
}
}
- public static String getIpAddress()
- throws SocketException, IOException
- {
- return getIpAddress(null, null, true, true);
- }
-
/**
* Examines each address associated with each network interface
* card (nic) installed on the current node, and returns the
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/counters/httpd/TestCounterSetHTTPDServer.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/counters/httpd/TestCounterSetHTTPDServer.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/counters/httpd/TestCounterSetHTTPDServer.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -39,6 +39,7 @@
import com.bigdata.counters.Instrument;
import com.bigdata.counters.OneShotInstrument;
import com.bigdata.counters.PeriodEnum;
+import com.bigdata.util.config.NicUtil;
/**
* Utility class for testing {@link CounterSetHTTPD} or
@@ -62,12 +63,10 @@
CounterSet cset = root.makePath("localhost");
- cset.addCounter("hostname", new OneShotInstrument<String>(
- InetAddress.getLocalHost().getHostName()));
+ String localIpAddr = NicUtil.getIpAddress("default.nic", "default", true);
+ cset.addCounter("hostname", new OneShotInstrument<String>(localIpAddr));
+ cset.addCounter("ipaddr", new OneShotInstrument<String>(localIpAddr));
- cset.addCounter("ipaddr", new OneShotInstrument<String>(InetAddress
- .getLocalHost().getHostAddress()));
-
// 60 minutes of data : @todo replace with CounterSetBTree (no fixed limit).
final HistoryInstrument<Double> history1 = new HistoryInstrument<Double>(
new History<Double>(new Double[60], PeriodEnum.Minutes
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/service/TestResourceService.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/service/TestResourceService.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/service/TestResourceService.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -43,6 +43,7 @@
import com.bigdata.service.ResourceService.ReadResourceTask;
import com.bigdata.util.concurrent.DaemonThreadFactory;
+import com.bigdata.util.config.NicUtil;
/**
* Test verifies the ability to transmit a file using the
@@ -103,14 +104,15 @@
};
+ InetAddress thisInetAddr;
+ thisInetAddr = InetAddress.getByName(NicUtil.getIpAddress("default.nic", "default", true));
try {
service.awaitRunning(100, TimeUnit.MILLISECONDS);
assertTrue(service.isOpen());
- assertEquals(tmpFile, new ReadResourceTask(InetAddress
- .getLocalHost(), service.port, allowedUUID, tmpFile).call());
+ assertEquals(tmpFile, new ReadResourceTask(thisInetAddr, service.port, allowedUUID, tmpFile).call());
if (log.isInfoEnabled())
log.info(service.counters.getCounters());
@@ -182,6 +184,8 @@
final List<File> tempFiles = new LinkedList<File>();
+ InetAddress thisInetAddr;
+ thisInetAddr = InetAddress.getByName(NicUtil.getIpAddress("default.nic", "default", true));
try {
service.awaitRunning(100, TimeUnit.MILLISECONDS);
@@ -197,8 +201,7 @@
tempFiles.add(tmpFile);
- tasks.add(new ReadResourceTask(InetAddress.getLocalHost(),
- service.port, allowedUUID, tmpFile));
+ tasks.add(new ReadResourceTask(thisInetAddr, service.port, allowedUUID, tmpFile));
}
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/test/ExperimentDriver.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/test/ExperimentDriver.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/test/ExperimentDriver.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -68,6 +68,7 @@
import com.bigdata.journal.ProxyTestCase;
import com.bigdata.util.NV;
+import com.bigdata.util.config.NicUtil;
/**
* A harness for running comparison of different configurations.
@@ -1364,13 +1365,9 @@
props.setProperty("os.arch.cpus", ""+SystemUtil.numProcessors());
try {
-
- props.setProperty("host",InetAddress.getLocalHost().getHostName());
-
- } catch(UnknownHostException ex) {
- /*
- * ignore.
- */
+ props.setProperty( "host", NicUtil.getIpAddress("default.nic", "default", true) );
+ } catch(Throwable t) {
+ t.printStackTrace();
}
return props;
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/ServiceImpl.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/ServiceImpl.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/ServiceImpl.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -244,13 +244,9 @@
}
innerProxy = (PrivateInterface)serverExporter.export(this);
+ String hostname = NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
outerProxy = ServiceProxy.createProxy
- (innerProxy, proxyId,
- NicUtil.getIpAddress(
- System.getProperty(
- "exportNic",
- ConfigDeployUtil.getString(
- "node.serviceNetwork"))) );
+ (innerProxy, proxyId, hostname);
adminProxy = AdminProxy.createProxy(innerProxy, proxyId);
//Setup lookup discovery
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/config/executor.config
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/config/executor.config 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/config/executor.config 2010-07-28 20:47:53 UTC (rev 3337)
@@ -16,10 +16,8 @@
com.bigdata.executor {
- private static exportNic =
- System.getProperty("exportNic",
- ConfigDeployUtil.getString("node.serviceNetwork"));
- private static exportIpAddr = NicUtil.getIpAddress(exportNic);
+ private static exportIpAddr =
+ NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
private static exportPort =
Integer.parseInt( System.getProperty("exportPort", "0") );
@@ -59,7 +57,7 @@
net.jini.discovery.LookupDiscovery {
multicastRequestHost = com.bigdata.executor.exportIpAddr;
multicastInterfaces = new NetworkInterface[] {
- NicUtil.getNetworkInterface(com.bigdata.executor.exportNic)
+ NicUtil.getNetworkInterface(com.bigdata.executor.exportIpAddr)
};
}
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/JiniServiceConfiguration.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/JiniServiceConfiguration.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/JiniServiceConfiguration.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -78,6 +78,8 @@
import com.bigdata.service.jini.JiniClient;
import com.bigdata.service.jini.JiniClientConfig;
import com.bigdata.service.jini.JiniFederation;
+import com.bigdata.util.config.ConfigDeployUtil;
+import com.bigdata.util.config.NicUtil;
import com.bigdata.zookeeper.ZNodeCreatedWatcher;
/**
@@ -130,6 +132,8 @@
public final Properties properties;
public final String[] jiniOptions;
+ private final String serviceIpAddr;
+
protected void toString(StringBuilder sb) {
super.toString(sb);
@@ -155,6 +159,7 @@
final Configuration config) throws ConfigurationException {
super(className, config);
+//BTM
System.out.println("*** JiniServiceConfiguration: constructor");
final JiniClientConfig tmp = new JiniClientConfig(className, config);
@@ -176,6 +181,12 @@
} else {
log.warn("groups = " + Arrays.toString(this.groups));
}
+
+ try {
+ this.serviceIpAddr = NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
+ } catch(IOException e) {
+ throw new ConfigurationException(e.getMessage(), e);
+ }
}
/**
@@ -472,8 +483,7 @@
final ServiceDir serviceDir = new ServiceDir(this.serviceDir);
- final Hostname hostName = new Hostname(InetAddress.getLocalHost()
- .getCanonicalHostName().toString());
+ final Hostname hostName = new Hostname(serviceIpAddr);
final ServiceUUID serviceUUID = new ServiceUUID(this.serviceUUID);
@@ -854,7 +864,8 @@
log.info("Discovered service: elapsed=" + elapsed
+ ", name=" + processHelper.name + ", item="
+ items[0]);
-System.out.println("\n**** JiniServiceConfiguration: DISCOVERED SERVICE ---> "+items[0].service+" ****\n");
+//BTM
+System.out.println("\n**** JiniServiceConfiguration.awaitServiceDiscoveryOrDeath: DISCOVERED ---> "+items[0].service+" ****");
return items[0];
} finally {
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/MaxClientServicesPerHostConstraint.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/MaxClientServicesPerHostConstraint.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/MaxClientServicesPerHostConstraint.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -10,6 +10,8 @@
import com.bigdata.jini.lookup.entry.ServiceItemFilterChain;
import com.bigdata.service.IClientService;
import com.bigdata.service.jini.JiniFederation;
+import com.bigdata.util.config.ConfigDeployUtil;
+import com.bigdata.util.config.NicUtil;
/**
* Constraint on the #of {@link IClientService}s on the same host.
@@ -51,11 +53,9 @@
// */
// filter.add(ClientServiceFilter.INSTANCE);
- final String hostname = InetAddress.getLocalHost().getHostName();
+ final String hostname = NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
+ final String canonicalHostname = hostname;
- final String canonicalHostname = InetAddress.getLocalHost()
- .getCanonicalHostName();
-
// filters for _this_ host.
filter.add(new HostnameFilter(new Hostname[] {//
new Hostname(hostname),//
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/MaxDataServicesPerHostConstraint.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/MaxDataServicesPerHostConstraint.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/MaxDataServicesPerHostConstraint.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -11,6 +11,8 @@
import com.bigdata.service.IDataService;
import com.bigdata.service.jini.JiniFederation;
import com.bigdata.service.jini.lookup.DataServiceFilter;
+import com.bigdata.util.config.ConfigDeployUtil;
+import com.bigdata.util.config.NicUtil;
/**
* Constraint on the #of {@link IDataService}s on the same host.
@@ -47,11 +49,9 @@
// only consider data services.
filter.add(DataServiceFilter.INSTANCE);
- final String hostname = InetAddress.getLocalHost().getHostName();
+ final String hostname = NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
+ final String canonicalHostname = hostname;
- final String canonicalHostname = InetAddress.getLocalHost()
- .getCanonicalHostName();
-
// filters for _this_ host.
filter.add(new HostnameFilter(new Hostname[] {//
new Hostname(hostname),//
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ZookeeperServerConfiguration.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ZookeeperServerConfiguration.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ZookeeperServerConfiguration.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -54,6 +54,8 @@
import com.bigdata.io.FileLockUtility;
import com.bigdata.jini.start.IServiceListener;
import com.bigdata.jini.start.process.ZookeeperProcessHelper;
+import com.bigdata.util.config.ConfigDeployUtil;
+import com.bigdata.util.config.NicUtil;
import com.bigdata.zookeeper.ZooHelper;
/**
@@ -213,6 +215,8 @@
*/
public final Map<String, String> other;
+ private final InetAddress thisInetAddr;
+
/**
* Adds value to {@link #other} if found in the {@link Configuration}.
*
@@ -318,6 +322,14 @@
putIfDefined(config, Options.FORCE_SYNC, Boolean.TYPE);
putIfDefined(config, Options.SKIP_ACL, Boolean.TYPE);
+ try {
+ thisInetAddr = InetAddress.getByName(NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false));
+ } catch(IOException e) {
+ throw new ConfigurationException(e.getMessage(), e);
+ }
+ if (log.isInfoEnabled()) {
+ log.info("zookeeper host="+thisInetAddr.getCanonicalHostName());
+ }
}
/**
@@ -541,7 +553,7 @@
*/
public V call() throws Exception {
- if (ZooHelper.isRunning(InetAddress.getLocalHost(), clientPort)) {
+ if (ZooHelper.isRunning(thisInetAddr, clientPort)) {
/*
* Query for an instance already running on local host at that
@@ -552,7 +564,7 @@
* instance on the localhost.
*/
- ZooHelper.ruok(InetAddress.getLocalHost(), clientPort);
+ ZooHelper.ruok(thisInetAddr, clientPort);
throw new ZookeeperRunningException(
"Zookeeper already running on localhost: clientport="
@@ -625,7 +637,7 @@
* Note: We don't test this until we have the file lock.
*/
- ZooHelper.ruok(InetAddress.getLocalHost(), clientPort);
+ ZooHelper.ruok(thisInetAddr, clientPort);
throw new ZookeeperRunningException(
"Zookeeper already running on localhost: clientport="
@@ -727,7 +739,7 @@
* clientPort. That could have already been true.
*/
- ZooHelper.ruok(InetAddress.getLocalHost(), clientPort);
+ ZooHelper.ruok(thisInetAddr, clientPort);
// adjust for time remaining.
nanos = (System.nanoTime() - begin);
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/JiniCoreServicesProcessHelper.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/JiniCoreServicesProcessHelper.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/JiniCoreServicesProcessHelper.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -42,6 +42,8 @@
import com.bigdata.service.jini.JiniClientConfig;
import com.bigdata.service.jini.util.JiniServicesHelper;
import com.bigdata.service.jini.util.LookupStarter;
+import com.bigdata.util.config.ConfigDeployUtil;
+import com.bigdata.util.config.NicUtil;
/**
* Class for starting the jini services.
@@ -83,12 +85,14 @@
public static boolean startCoreServices(final Configuration config,
final IServiceListener listener) throws Exception {
+//BTM
System.out.println("*** JiniCoreServicesProcessHelper.startCoreServices: new JiniCoreServicesConfiguration() ...");
final JiniCoreServicesConfiguration serviceConfig = new JiniCoreServicesConfiguration(
config);
// Note: will throw NPE or IAE if constraint relies on fed!
if (!serviceConfig.canStartService(null/* fed */)) {
+//BTM
System.out.println("*** JiniCoreServicesProcessHelper.startCoreServices: Constraints do NOT permit start --> "+serviceConfig);
// refuse to start.
@@ -97,11 +101,13 @@
}
+//BTM
System.out.println("*** JiniCoreServicesProcessHelper.startCoreServices: new JiniClientConfig() ...");
final JiniClientConfig clientConfig = new JiniClientConfig(
null/* class */, config);
final LookupLocator[] locators = clientConfig.locators;
+//BTM
System.out.println("*** JiniCoreServicesProcessHelper.startCoreServices: locators.length = "+locators.length);
if (locators.length > 0) {
@@ -120,6 +126,7 @@
int i = 0;
for (LookupLocator locator : locators) {
+//BTM
System.out.println("*** JiniCoreServicesProcessHelper.startCoreServices: locator["+i+"] = "+locator);
allowed[i++] = locator.getHost();
@@ -130,6 +137,7 @@
if (log.isInfoEnabled())
log.info("Host not selected by locator(s).");
+//BTM
System.out.println("*** JiniCoreServicesProcessHelper.startCoreServices: locator NOT ALLOWED");
return false;
@@ -144,6 +152,7 @@
if (log.isInfoEnabled())
log.info("Constraint(s) do not allow service start: " + config);
+//BTM
System.out.println("*** JiniCoreServicesProcessHelper.startCoreServices: CONSTRAINTS do not allow start ["+config+"]");
return false;
@@ -153,6 +162,7 @@
/*
* The #of registrars that we can locate within a timeout.
*/
+//BTM
System.out.println("*** JiniCoreServicesProcessHelper.startCoreServices: JiniServicesHelper.getServiceRegistrars() ...");
final ServiceRegistrar[] registrars = JiniServicesHelper
.getServiceRegistrars(Integer.MAX_VALUE/* maxCount */,
@@ -162,13 +172,13 @@
if (log.isInfoEnabled())
log.info("registrars: #found=" + registrars.length + ", #desired="
+ serviceConfig.serviceCount);
+//BTM
System.out.println("*** JiniCoreServicesProcessHelper.startCoreServices: JiniServicesHelper.getServiceRegistrars() - #found=" + registrars.length + ", #desired=" + serviceConfig.serviceCount);
if (registrars.length >= serviceConfig.serviceCount) {
if (log.isInfoEnabled())
log.info("Enough instances - will not start another.");
-System.out.println("*** JiniCoreServicesProcessHelper.startCoreServices: Enough instances - will not start another.");
return false;
@@ -179,17 +189,20 @@
*/
if (log.isInfoEnabled())
- log.info("Will start instance: " + InetAddress.getLocalHost()
+ log.info("Will start instance: " + NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false)
+ ", config=" + config);
+//BTM
System.out.println("*** JiniCoreServicesProcessHelper.startCoreServices: serviceConfig.newServiceStarter() ...");
final JiniCoreServicesStarter<JiniCoreServicesProcessHelper> serviceStarter = serviceConfig
.newServiceStarter(listener, clientConfig);
// start jini.
+//BTM
System.out.println("*** JiniCoreServicesProcessHelper.startCoreServices: serviceStarter.call() ...");
final JiniCoreServicesProcessHelper processHelper = serviceStarter
.call();
+//BTM
System.out.println("*** JiniCoreServicesProcessHelper.startCoreServices: serviceStarter.call() ... RETURNED!!!");
return true;
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/ZookeeperProcessHelper.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/ZookeeperProcessHelper.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/ZookeeperProcessHelper.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -16,6 +16,8 @@
import com.bigdata.jini.start.config.ZookeeperServerConfiguration;
import com.bigdata.jini.start.config.ZookeeperServerEntry;
import com.bigdata.jini.start.config.ZookeeperServerConfiguration.ZookeeperRunningException;
+import com.bigdata.util.config.ConfigDeployUtil;
+import com.bigdata.util.config.NicUtil;
import com.bigdata.zookeeper.ZooHelper;
/**
@@ -40,6 +42,14 @@
protected final int clientPort;
+ protected static InetAddress thisInetAddr = null;
+ static {
+ try {
+ thisInetAddr = InetAddress.getByName
+ (NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false));
+ } catch (Throwable t) { /* swallow */ }
+ }
+
/**
* @param name
* @param builder
@@ -144,14 +154,16 @@
final ZookeeperServerConfiguration serverConfig = new ZookeeperServerConfiguration(
config);
-System.out.println("---- ZookeeperProcessHelper.startZookeeper: [localhost="+InetAddress.getLocalHost()+", clientPort="+serverConfig.clientPort+"] ----");
- if (ZooHelper.isRunning(InetAddress.getLocalHost(), serverConfig.clientPort)) {
+//BTM
+System.out.println("---- ZookeeperProcessHelper.startZookeeper: [localhost="+thisInetAddr.getCanonicalHostName()+", clientPort="+serverConfig.clientPort+"] ----");
+ if (ZooHelper.isRunning(thisInetAddr, serverConfig.clientPort)) {
if (log.isInfoEnabled())
log.info("Zookeeper already running: "
- + InetAddress.getLocalHost().getCanonicalHostName()
+ + thisInetAddr.getCanonicalHostName()
+ ":" + serverConfig.clientPort);
-System.out.println("---- ZookeeperProcessHelper.startZookeeper: Zookeeper ALREADY RUNNING ----");
+//BTM
+System.out.println("---- ZookeeperProcessHelper.startZookeeper: Zookeeper ALREADY RUNNING on "+thisInetAddr.getCanonicalHostName()+" ----");
// will not consider start.
return 0;
@@ -179,7 +191,6 @@
for (ZookeeperServerEntry entry : entries) {
if (entry.isLocalHost()) {
-System.out.println("---- ZookeeperProcessHelper.startZookeeper: IS LOCAL HOST ----");
if (log.isInfoEnabled())
log.info("Zookeeper instance is local: " + entry);
@@ -203,8 +214,6 @@
log.error("Could not start: entry=" + entry, ex);
}
-}else{
-System.out.println("---- ZookeeperProcessHelper.startZookeeper: is NOT local host ----");
}
} // next entry
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/ServiceImpl.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/ServiceImpl.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/ServiceImpl.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -123,12 +123,12 @@
//To start service using the old bigdata ServicesManagerService mechanism or from the command line or tests without the Jini ServiceStarter
public static void main(String[] args) {
-String str1 = "/raglan/service/com.bigdata.jini.start.IServicesManagerService/61e99f71-a6f7-4a61-a330-ce7a04e6a0fb/Thread Pool/Service Time";
-String str2 = "/raglan/service/com.bigdata.jini.start.IServicesManagerService/61e99f71-a6f7-4a61-a330-ce7a04e6a0fb";
-String str3 = "/raglan/service/com.bigdata.jini.start.IServicesManagerService/";
-System.out.println("\n\n SSSSS >>>>> "+str1+ "str1.contains(//) >>>> "+str1.contains("//"));
-System.out.println("\n\n SSSSS >>>>> "+str2+ "str1.contains(//) >>>> "+str2.contains("//"));
-System.out.println("\n\n SSSSS >>>>> "+str3+ "str1.contains(//) >>>> "+str3.contains("//"));
+//String str1 = "/raglan/service/com.bigdata.jini.start.IServicesManagerService/61e99f71-a6f7-4a61-a330-ce7a04e6a0fb/Thread Pool/Service Time";
+//String str2 = "/raglan/service/com.bigdata.jini.start.IServicesManagerService/61e99f71-a6f7-4a61-a330-ce7a04e6a0fb";
+//String str3 = "/raglan/service/com.bigdata.jini.start.IServicesManagerService/";
+//System.out.println("\n\n SSSSS >>>>> "+str1+ "str1.contains(//) >>>> "+str1.contains("//"));
+//System.out.println("\n\n SSSSS >>>>> "+str2+ "str1.contains(//) >>>> "+str2.contains("//"));
+//System.out.println("\n\n SSSSS >>>>> "+str3+ "str1.contains(//) >>>> "+str3.contains("//"));
try {
ArrayList<String> argsList = new ArrayList<String>();
@@ -416,17 +416,17 @@
( args,
(this.getClass()).getClassLoader() );
if(smsProxyId == null) {//service assigns its own proxyId
-System.out.println("XXXXX LOAD BALANCER ServiceImpl: smsProxyId = null ---> service-assigned id");
+logger.warn("XXXXX LOAD BALANCER ServiceImpl: smsProxyId = null ---> service-assigned id");
BootStateUtil bootStateUtil =
new BootStateUtil(config, COMPONENT_NAME, this.getClass(), logger);
proxyId = bootStateUtil.getProxyId();
serviceId = bootStateUtil.getServiceId();
} else {//ServicesManagerService assigned the proxyId
-System.out.println("XXXXX LOAD BALANCER ServiceImpl: smsProxyId NOT null ---> SMS-assigned id");
+logger.warn("XXXXX LOAD BALANCER ServiceImpl: smsProxyId NOT null ---> SMS-assigned id");
proxyId = smsProxyId;
serviceId = com.bigdata.jini.util.JiniUtil.uuid2ServiceID(proxyId);
}
-System.out.println("XXXXX LOAD BALANCER ServiceImpl: proxyId = "+proxyId+"\n");
+logger.warn("XXXXX LOAD BALANCER ServiceImpl: proxyId = "+proxyId+"\n");
//Service export and proxy creation
ServerEndpoint endpoint = TcpServerEndpoint.getInstance(0);
@@ -445,11 +445,7 @@
}
innerProxy = (PrivateInterface)serverExporter.export(this);
- String hostname = NicUtil.getIpAddress(
- System.getProperty(
- "exportNic",
- ConfigDeployUtil.getString(
- "node.serviceNetwork")));
+ String hostname = NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
outerProxy = ServiceProxy.createProxy
(innerProxy, proxyId, hostname);
adminProxy = AdminProxy.createProxy(innerProxy, proxyId);
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/config/loadbalancer.config
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/config/loadbalancer.config 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/config/loadbalancer.config 2010-07-28 20:47:53 UTC (rev 3337)
@@ -16,10 +16,8 @@
com.bigdata.loadbalancer {
- private static exportNic =
- System.getProperty("exportNic",
- ConfigDeployUtil.getString("node.serviceNetwork"));
- private static exportIpAddr = NicUtil.getIpAddress(exportNic);
+ private static exportIpAddr =
+ NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
private static exportPort =
Integer.parseInt( System.getProperty("exportPort", "0") );
@@ -59,7 +57,7 @@
net.jini.discovery.LookupDiscovery {
multicastRequestHost = com.bigdata.loadbalancer.exportIpAddr;
multicastInterfaces = new NetworkInterface[] {
- NicUtil.getNetworkInterface(com.bigdata.loadbalancer.exportNic)
+ NicUtil.getNetworkInterface(com.bigdata.loadbalancer.exportIpAddr)
};
}
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/ServiceImpl.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/ServiceImpl.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/ServiceImpl.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -296,13 +296,9 @@
}
innerProxy = (PrivateInterface)serverExporter.export(this);
+ String hostname = NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
outerProxy = ServiceProxy.createProxy
- (innerProxy, proxyId,
- NicUtil.getIpAddress(
- System.getProperty(
- "exportNic",
- ConfigDeployUtil.getString(
- "node.serviceNetwork"))) );
+ (innerProxy, proxyId, hostname);
adminProxy = AdminProxy.createProxy(innerProxy, proxyId);
//Setup lookup discovery
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/shardlocator.config
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/shardlocator.config 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/shardlocator.config 2010-07-28 20:47:53 UTC (rev 3337)
@@ -16,10 +16,8 @@
com.bigdata.metadata {
- private static exportNic =
- System.getProperty("exportNic",
- ConfigDeployUtil.getString("node.serviceNetwork"));
- private static exportIpAddr = NicUtil.getIpAddress(exportNic);
+ private static exportIpAddr =
+ NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
private static exportPort =
Integer.parseInt( System.getProperty("exportPort", "0") );
@@ -59,7 +57,7 @@
net.jini.discovery.LookupDiscovery {
multicastRequestHost = com.bigdata.metadata.exportIpAddr;
multicastInterfaces = new NetworkInterface[] {
- NicUtil.getNetworkInterface(com.bigdata.metadata.exportNic)
+ NicUtil.getNetworkInterface(com.bigdata.metadata.exportIpAddr)
};
}
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/process/ServiceImpl.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/process/ServiceImpl.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/process/ServiceImpl.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -362,13 +362,9 @@
}
innerProxy = (PrivateInterface)serverExporter.export(this);
+ String hostname = NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
outerProxy = ServiceProxy.createProxy
- (innerProxy, proxyId,
- NicUtil.getIpAddress(
- System.getProperty(
- "exportNic",
- ConfigDeployUtil.getString(
- "node.serviceNetwork"))) );
+ (innerProxy, proxyId, hostname);
adminProxy = AdminProxy.createProxy(innerProxy, proxyId);
// Setup lookup discovery
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/process/config/process.config
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/process/config/process.config 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/process/config/process.config 2010-07-28 20:47:53 UTC (rev 3337)
@@ -16,10 +16,8 @@
com.bigdata.process {
- private static exportNic =
- System.getProperty("exportNic",
- ConfigDeployUtil.getString("node.serviceNetwork"));
- private static exportIpAddr = NicUtil.getIpAddress(exportNic);
+ private static exportIpAddr =
+ NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
private static exportPort =
Integer.parseInt( System.getProperty("exportPort", "0") );
@@ -59,7 +57,7 @@
net.jini.discovery.LookupDiscovery {
multicastRequestHost = com.bigdata.process.exportIpAddr;
multicastInterfaces = new NetworkInterface[] {
- NicUtil.getNetworkInterface(com.bigdata.process.exportNic)
+ NicUtil.getNetworkInterface(com.bigdata.process.exportIpAddr)
};
}
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/ServiceImpl.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/ServiceImpl.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/ServiceImpl.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -1092,12 +1092,7 @@
// servers are discovered and a leader is elected.
//zookeeperNetwork (peerAddress)
- String zookeeperNetwork =
- NicUtil.getIpAddress(
- System.getProperty(
- "exportNic",
- ConfigDeployUtil.getString(
- "node.serviceNetwork")));
+ String zookeeperNetwork = NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
if(zookeeperNetwork == null) {
throw new ConfigurationException
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/config/quorum.config
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/config/quorum.config 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/config/quorum.config 2010-07-28 20:47:53 UTC (rev 3337)
@@ -21,10 +21,8 @@
com.bigdata.quorum {
- private static exportNic =
- System.getProperty("exportNic",
- ConfigDeployUtil.getString("node.serviceNetwork"));
- private static exportIpAddr = NicUtil.getIpAddress(exportNic);
+ private static exportIpAddr =
+ NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
private static exportPort =
Integer.parseInt( System.getProperty("exportPort", "0") );
@@ -78,7 +76,7 @@
net.jini.discovery.LookupDiscovery {
multicastRequestHost = com.bigdata.quorum.exportIpAddr;
multicastInterfaces = new NetworkInterface[] {
- NicUtil.getNetworkInterface(com.bigdata.quorum.exportNic)
+ NicUtil.getNetworkInterface(com.bigdata.quorum.exportIpAddr)
};
}
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/LoadBalancerServer.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/LoadBalancerServer.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/LoadBalancerServer.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -36,6 +36,8 @@
import com.bigdata.service.jini.util.DumpFederation;
import com.bigdata.service.jini.util.DumpFederation.FormatRecord;
import com.bigdata.service.jini.util.DumpFederation.FormatTabTable;
+import com.bigdata.util.config.ConfigDeployUtil;
+import com.bigdata.util.config.NicUtil;
import com.bigdata.util.httpd.AbstractHTTPD;
import com.bigdata.util.httpd.NanoHTTPD;
import com.bigdata.util.httpd.NanoHTTPD.Response;
@@ -43,6 +45,7 @@
import com.sun.jini.start.ServiceDescriptor;
import com.sun.jini.start.ServiceStarter;
+
/**
* The load balancer server.
* <p>
@@ -634,15 +637,10 @@
* This exception gets thrown if the client has made a direct
* (vs RMI) call.
*/
-
try {
-
- clientAddr = Inet4Address.getLocalHost();
-
- } catch (UnknownHostException ex) {
-
- return "localhost";
-
+ clientAddr = InetAddress.getByName(NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.dataNetwork"), false));
+ } catch(Throwable t) {//for now, maintain the same failure logic as used previously
+ return NicUtil.getIpAddressByLocalHost();
}
}
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/LookupStarter.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/LookupStarter.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/LookupStarter.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -74,8 +74,7 @@
private static String defaultGroup = null;
static {
try {
- thisHost = NicUtil.getIpAddress
- ("default.nic", null, true, true);
+ thisHost = NicUtil.getIpAddress("default.nic", "default", true);
defaultGroup =
System.getProperty("federation.name",
"bigdata.test.group-"+thisHost);
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/config/lookup.config
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/config/lookup.config 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/config/lookup.config 2010-07-28 20:47:53 UTC (rev 3337)
@@ -10,7 +10,7 @@
com.sun.jini.reggie {
private static exportPort = 0;//randomly chosen port
private static exportIpAddr =
- NicUtil.getIpAddress("default.nic", null, true, true);
+ NicUtil.getIpAddress("default.nic", "default", true);
private static codebasePort = 23333;//value used if not set by system prop
// Public configuration entries
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/ServiceImpl.java
==============================================...
[truncated message content] |
|
From: <btm...@us...> - 2010-07-29 12:58:08
|
Revision: 3340
http://bigdata.svn.sourceforge.net/bigdata/?rev=3340&view=rev
Author: btmurphy
Date: 2010-07-29 12:57:54 +0000 (Thu, 29 Jul 2010)
Log Message:
-----------
merge -r:3267:HEAD(3339) ~/trunk/bigdata ~/bigdata/branches/dev-btm [trunk --> branch dev-btm]
Modified Paths:
--------------
branches/dev-btm/.classpath
branches/dev-btm/bigdata/src/architecture/mergePriority.xls
branches/dev-btm/bigdata/src/java/com/bigdata/LRUNexus.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/BloomFilterFactory.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/Checkpoint.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/DefaultTupleSerializer.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexMetadata.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegment.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentBuilder.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentStore.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/NOPTupleSerializer.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/NodeSerializer.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/isolation/IsolatedFusedView.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/IKeyBuilder.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/proc/AbstractKeyArrayIndexProcedure.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/view/FusedView.java
branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java
branches/dev-btm/bigdata/src/java/com/bigdata/io/DirectBufferPool.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractBufferStrategy.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractJournal.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/BufferMode.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/CommitRecordIndex.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/FileMetadata.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/Name2Addr.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/Options.java
branches/dev-btm/bigdata/src/java/com/bigdata/mdi/IndexPartitionCause.java
branches/dev-btm/bigdata/src/java/com/bigdata/mdi/MetadataIndex.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/IRelation.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/accesspath/SameVariableConstraint.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/locator/DefaultResourceLocator.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/IPredicate.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/Rule.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/Var.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/DefaultRuleTaskFactory.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/RuleState.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/JoinTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/LocalJoinTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/JournalIndex.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/StoreManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/ViewMetadata.java
branches/dev-btm/bigdata/src/java/com/bigdata/search/ReadIndexTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/CommitTimeIndex.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ILoadBalancerService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/LoadBalancerService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/RawDataServiceTupleIterator.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/proxy/RemoteAsynchronousIteratorImpl.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/proxy/RemoteChunk.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/AbstractAtomicRowReadOrWrite.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/TPSTupleSerializer.java
branches/dev-btm/bigdata/src/java/com/bigdata/striterator/IKeyOrder.java
branches/dev-btm/bigdata/src/resources/logging/log4j.properties
branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractBTreeTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractIndexSegmentTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestAll_IndexSegment.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestBigdataMap.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexSegmentBuilderWithLargeTrees.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexSegmentBuilderWithSmallTree.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestLeafSplitShortestSeparatorKey.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/AbstractUnicodeKeyBuilderTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java
branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestAll.java
branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestRootBlockView.java
branches/dev-btm/bigdata/src/test/com/bigdata/relation/accesspath/TestSameVariableConstraint.java
branches/dev-btm/bigdata/src/test/com/bigdata/relation/rule/AbstractRuleTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/relation/rule/TestRule.java
branches/dev-btm/bigdata/src/test/com/bigdata/search/TestAll.java
branches/dev-btm/bigdata/src/test/com/bigdata/search/TestSearchRestartSafe.java
branches/dev-btm/bigdata/src/test/com/bigdata/sparse/TestAll.java
branches/dev-btm/bigdata/src/test/com/bigdata/util/concurrent/TestAll.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/AbstractServicesManagerService.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServicesManagerServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/AbstractServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/LoadBalancerServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/benchmark/ThroughputMaster.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/BroadcastSighup.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/axioms/Axioms.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/axioms/BaseAxioms.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/BackchainOwlSameAsIterator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/BackchainOwlSameAsPropertiesIterator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/BackchainOwlSameAsPropertiesPIterator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/BackchainOwlSameAsPropertiesPOIterator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/BackchainOwlSameAsPropertiesSPIterator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/BackchainOwlSameAsPropertiesSPOIterator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/BackchainTypeResourceIterator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/FullyBufferedJustificationIterator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/Justification.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/OwlSameAsPropertiesExpandingIterator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/RdfTypeRdfsResourceFilter.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/SPOAssertionBuffer.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/TruthMaintenance.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/BigdataRDFFullTextIndex.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/ITermIndexCodes.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Id2TermTupleSerializer.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Id2TermWriteProc.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/KVOTermIdComparator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconKeyBuilder.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/ReverseIndexWriterTask.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdTupleSerializer.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdWriteProc.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdWriteTask.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/TermIdEncoder.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/VerifyStatementBuffer.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/magic/IMagicTuple.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/magic/IRISUtils.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/magic/MagicAccessPath.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/magic/MagicIndexWriteProc.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/magic/MagicKeyOrder.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/magic/MagicPredicate.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/magic/MagicRelation.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/magic/MagicTuple.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/magic/MagicTupleSerializer.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/BNodeContextFactory.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataBNodeImpl.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataLiteralImpl.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataResourceImpl.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataStatementImpl.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataURIImpl.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataValue.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataValueFactory.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataValueFactoryImpl.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataValueIdComparator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataValueImpl.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/StatementEnum.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/TermIdComparator2.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/NQuadsParser.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/StatementBuffer.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/AbstractRuleDistinctTermScan.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/AbstractRuleFastClosure_11_13.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/AbstractRuleFastClosure_3_5_6_7_9.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/AbstractRuleFastClosure_5_6_7_9.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/BackchainAccessPath.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/DoNotAddFilter.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/FastClosure.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/MatchRule.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/RDFJoinNexus.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/RuleFastClosure3.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/DefaultGraphSolutionExpander.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/DistinctMultiTermAdvancer.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/DistinctSPOIterator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/DistinctTermAdvancer.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/ExplicitSPOFilter.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/ISPO.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/InGraphBinarySearchFilter.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/InGraphHashSetFilter.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/InferredSPOFilter.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/JustificationRemover.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/JustificationTupleSerializer.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/NamedGraphSolutionExpander.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/NoAxiomFilter.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/OSPComparator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/POSComparator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPO.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOAccessPath.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOComparator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOFilter.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOIndexWriteProc.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOKeyOrder.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOPredicate.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPORelation.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOSortKeyBuilder.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOStarJoin.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOTupleSerializer.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/BigdataSolutionResolverator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/BigdataStatementIteratorImpl.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/BigdataValueIteratorImpl.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/IRawTripleStore.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/ITripleStore.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/ScaleOutTripleStore.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/TripleStoreUtility.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/vocab/BaseVocabulary.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/vocab/Vocabulary.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/TestAll.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/axioms/TestAxioms.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestAddTerms.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestAll.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestComparators.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestCompletionScan.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestFullTextIndex.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestId2TermTupleSerializer.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestTerm2IdTupleSerializer.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestVocabulary.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/magic/TestIRIS.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/magic/TestMagicKeyOrderStrategy.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/magic/TestMagicStore.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/model/TestAll.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/TestRDFXMLInterchangeWithStatementIdentifiers.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestAll.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestBackchainOwlSameAsPropertiesIterator.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestBackchainTypeResourceIterator.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestDatabaseAtOnceClosure.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestDistinctTermScan.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestJustifications.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestOptionals.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestOwlSameAsPropertiesExpandingIterator.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestRuleExpansion.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestRuleFastClosure_11_13.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestRuleFastClosure_3_5_6_7_9.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestSlice.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestTruthMaintenance.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestDefaultGraphAccessPath.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPO.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOAccessPath.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOKeyCoders.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOKeyOrder.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOPredicate.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPORelation.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOStarJoin.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOTupleSerializer.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOValueCoders.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/AbstractDistributedTripleStoreTestCase.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/AbstractEmbeddedTripleStoreTestCase.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/AbstractTestCase.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/TestAll.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/TestBulkFilter.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/TestLocalTripleStoreTransactionSemantics.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/TestRestartSafe.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/TestStatementIdentifiers.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/TestTripleStore.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/TestTripleStoreBasics.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataConstructIterator.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStatistics.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl2.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailHelper.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailRepository.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailRepositoryConnection.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/FreeTextSearchExpander.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/HitConvertor.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlClient.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestAll.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailEvaluationStrategyImpl.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestJoinScope.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestNamedGraphs.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestProvenanceQuery.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestQuery.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestSetBinding.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest2.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/stress/LoadClosureAndQueryTest.java
branches/dev-btm/bigdata-sails/src/test/org/openrdf/query/parser/sparql/SPARQLQueryTest.java
branches/dev-btm/build.properties
branches/dev-btm/src/resources/config/bigdataCluster.config
branches/dev-btm/src/resources/config/bigdataCluster16.config
branches/dev-btm/src/resources/config/standalone/bigdataStandalone.config
branches/dev-btm/src/resources/scripts/archiveRun.sh
branches/dev-btm/src/resources/scripts/bigdata
branches/dev-btm/src/resources/scripts/extractCounters.sh
Added Paths:
-----------
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentMultiBlockIterator.java
branches/dev-btm/bigdata/src/releases/RELEASE_0_83_0.txt
branches/dev-btm/bigdata/src/releases/RELEASE_0_83_1.txt
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexSegmentMultiBlockIterators.java
branches/dev-btm/bigdata-perf/bsbm/
branches/dev-btm/bigdata-perf/bsbm/RWStore.properties
branches/dev-btm/bigdata-perf/bsbm/WORMStore.properties
branches/dev-btm/bigdata-perf/bsbm/build.properties
branches/dev-btm/bigdata-perf/bsbm/build.xml
branches/dev-btm/bigdata-perf/bsbm/lib/
branches/dev-btm/bigdata-perf/bsbm/lib/jdom.jar
branches/dev-btm/bigdata-perf/bsbm/lib/log4j-1.2.12.jar
branches/dev-btm/bigdata-perf/bsbm/lib/ssj.jar
branches/dev-btm/bigdata-perf/bsbm/src/
branches/dev-btm/bigdata-perf/bsbm/src/java/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/DateGenerator.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/Generator.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/NormalDistGenerator.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/NormalDistRangeGenerator.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/ParetoDistGenerator.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/RandomBucket.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/TextGenerator.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/ValueGenerator.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/BSBMResource.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/Offer.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/Person.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/Producer.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/Product.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/ProductFeature.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/ProductType.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/RatingSite.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/Review.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/Vendor.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/qualification/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/qualification/Qualification.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/qualification/QualificationDefaultValues.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/qualification/QueryResult.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/NTriples.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/ObjectBundle.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/SQLSerializer.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/Serializer.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/TriG.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/Turtle.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/VirtSerializer.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/XMLSerializer.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/AbstractParameterPool.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/ClientManager.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/ClientThread.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/CompiledQuery.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/CompiledQueryMix.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/LocalSPARQLParameterPool.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/NetQuery.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/PreCalcParameterPool.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/Query.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/QueryMix.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/SPARQLConnection.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/SQLConnection.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/SQLParameterPool.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/ServerConnection.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/TestDriver.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/TestDriverDefaultValues.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/tools/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/tools/ResultTransform.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/BSBM.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/DC.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/FOAF.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/ISO3166.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/RDF.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/RDFS.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/REV.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/XSD.java
branches/dev-btm/bigdata-perf/bsbm/src/resources/
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/givennames.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/ignoreQueries.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query1.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query10.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query10desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query10valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query11.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query11desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query11valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query12.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query12desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query12valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query1desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query1valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query2.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query2desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query2valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query3.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query3desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query3valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query4-original.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query4-rewritten.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query4.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query4desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query4valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query5.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query5desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query5valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query6.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query6desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query6valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query7.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query7desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query7valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query8.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query8desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query8valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query9-original.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query9.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query9desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query9valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/querymix.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/titlewords.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/logging/
branches/dev-btm/bigdata-perf/bsbm/src/resources/logging/log4j.properties
branches/dev-btm/bigdata-perf/bsbm/src/test/
branches/dev-btm/bigdata-perf/bsbm/src/test/benchmark/
branches/dev-btm/bigdata-perf/bsbm/src/test/benchmark/bigdata/
branches/dev-btm/bigdata-perf/bsbm/src/test/benchmark/bigdata/TestBSBM.java
branches/dev-btm/bigdata-perf/lubm/
branches/dev-btm/bigdata-perf/lubm/LEGAL/
branches/dev-btm/bigdata-perf/lubm/LEGAL/LICENSE.txt
branches/dev-btm/bigdata-perf/lubm/README.txt
branches/dev-btm/bigdata-perf/lubm/RWStore.properties
branches/dev-btm/bigdata-perf/lubm/WORMStore.properties
branches/dev-btm/bigdata-perf/lubm/build.properties
branches/dev-btm/bigdata-perf/lubm/build.xml
branches/dev-btm/bigdata-perf/lubm/lib/
branches/dev-btm/bigdata-perf/lubm/src/
branches/dev-btm/bigdata-perf/lubm/src/java/
branches/dev-btm/bigdata-perf/lubm/src/java/edu/
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/uba/
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/uba/CompressEnum.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/uba/DamlWriter.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/uba/Generator.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/uba/OwlWriter.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/uba/RdfWriter.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/uba/Writer.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/uba/readme.txt
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/ConfigParser.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/KbConfigParser.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/KbSpecification.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/QueryConfigParser.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/QuerySpecification.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/QueryTestResult.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/RepositoryCreator.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/Test.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/api/
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/api/Atom.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/api/Query.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/api/QueryResult.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/api/Repository.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/api/RepositoryFactory.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/bigdata/
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/bigdata/SparqlRepository.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/bigdata/SparqlRepositoryFactory.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/config.kb.example.dldb
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/config.kb.example.sesame
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/config.query.example.dldb
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/config.query.example.sesame
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/readme.txt
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/univ-bench.owl
branches/dev-btm/bigdata-perf/lubm/src/resources/
branches/dev-btm/bigdata-perf/lubm/src/resources/answers (U1)/
branches/dev-btm/bigdata-perf/lubm/src/resources/answers (U1)/answers_query14.txt
branches/dev-btm/bigdata-perf/lubm/src/resources/answers (U1)/answers_query6.txt
branches/dev-btm/bigdata-perf/lubm/src/resources/answers (U1)/answers_query8.txt
branches/dev-btm/bigdata-perf/lubm/src/resources/answers (U1)/reference query answers.url
branches/dev-btm/bigdata-perf/lubm/src/resources/config/
branches/dev-btm/bigdata-perf/lubm/src/resources/config/config.kb.sparql
branches/dev-btm/bigdata-perf/lubm/src/resources/config/config.query.sparql
branches/dev-btm/bigdata-perf/lubm/src/resources/config/config.query1.sparql
branches/dev-btm/bigdata-perf/lubm/src/resources/config/config.query10.sparql
branches/dev-btm/bigdata-perf/lubm/src/resources/config/config.query14.sparql
branches/dev-btm/bigdata-perf/lubm/src/resources/config/config.query2.sparql
branches/dev-btm/bigdata-perf/lubm/src/resources/config/config.query3.sparql
branches/dev-btm/bigdata-perf/lubm/src/resources/config/config.query4.sparql
branches/dev-btm/bigdata-perf/lubm/src/resources/config/config.query6.sparql
branches/dev-btm/bigdata-perf/lubm/src/resources/config/config.query8-9.sparql
branches/dev-btm/bigdata-perf/lubm/src/resources/config/config.query8.sparql
branches/dev-btm/bigdata-perf/lubm/src/resources/config/config.query9.sparql
branches/dev-btm/bigdata-perf/lubm/src/resources/config/config.queryJDS.sparql
branches/dev-btm/bigdata-perf/lubm/src/resources/config/config.query_fastOnly.sparql
branches/dev-btm/bigdata-perf/lubm/src/resources/config/config.query_slowOnly.sparql
branches/dev-btm/bigdata-perf/lubm/src/resources/config/univ-bench.owl
branches/dev-btm/bigdata-perf/lubm/src/resources/logging/
branches/dev-btm/bigdata-perf/lubm/src/resources/logging/log4j.properties
branches/dev-btm/bigdata-perf/lubm/src/resources/scripts/
branches/dev-btm/bigdata-perf/lubm/src/resources/scripts/lubmClientGen.sh
branches/dev-btm/bigdata-perf/lubm/src/resources/scripts/lubmGen.sh
branches/dev-btm/bigdata-perf/lubm/src/resources/scripts/lubmMaster.sh
branches/dev-btm/bigdata-perf/lubm/src/resources/scripts/lubmQuery.sh
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractBNodeIV.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractIV.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractInlineIV.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/AbstractLiteralIV.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/DTE.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/DTEFlags.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/DefaultExtensionFactory.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/DummyIV.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/ExtensionIV.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/IDatatypeURIResolver.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtension.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtensionFactory.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/ILexiconConfiguration.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/IV.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/NullIV.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/NumericBNodeIV.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/TermId.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/UUIDBNodeIV.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/UUIDLiteralIV.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/VTE.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSD.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDBooleanIV.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDByteIV.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDDecimalIV.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDDoubleIV.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDFloatIV.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDIntIV.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDIntegerIV.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDLongIV.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDShortIV.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/AbstractInlineConstraint.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/InlineEQ.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/InlineGE.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/InlineGT.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/InlineLE.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/InlineLT.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/InlineNE.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/package.html
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/ITermIVFilter.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/TermIVComparator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/relation/
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/relation/rule/
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/relation/rule/BindingSetSortKeyBuilder.java
branches/dev-btm/bigdata-rdf/src/samples/
branches/dev-btm/bigdata-rdf/src/samples/com/
branches/dev-btm/bigdata-rdf/src/samples/com/bigdata/
branches/dev-btm/bigdata-rdf/src/samples/com/bigdata/rdf/
branches/dev-btm/bigdata-rdf/src/samples/com/bigdata/rdf/internal/
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/internal/
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/internal/BlobOverflowHandler.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/internal/ColorsEnumExtension.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/internal/EpochExtension.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/internal/ITermIdCodes.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/internal/LegacyTermIdUtility.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/internal/SampleExtensionFactory.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestAll.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestEncodeDecodeKeys.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestIVCompare.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestLongLiterals.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/internal/constraints/
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/internal/constraints/TestAll.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/internal/constraints/TestInlineConstraints.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestInlining.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/magic/TestAll.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/model/TestFactory.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/relation/
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/relation/rule/
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/TestLocalTripleStoreWithoutInlining.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuadsAndPipelineJoinsWithoutInlining.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithSidsWithoutInlining.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestInlineValues.java
branches/dev-btm/src/resources/scripts/broadcast_sighup
Removed Paths:
-------------
branches/dev-btm/bigdata/src/architecture/scale-out-perf.txt
branches/dev-btm/bigdata/src/java/com/bigdata/btree/AddressSerializer.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IAddressSerializer.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/ISplitHandler.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/PackedAddressSerializer.java
branches/dev-btm/bigdata/src/java/com/bigdata/mdi/ReadOnlyMetadataIndexView.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/BindingSetSortKeyBuilder.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/DefaultSplitHandler.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/LocalDataServiceClient.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/LocalDataServiceFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/DataServiceIndex.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestLDS.java
branches/dev-btm/bigdata/src/test/com/bigdata/util/concurrent/ParallelismLimitedExecutorService.java
branches/dev-btm/bigdata/src/test/com/bigdata/util/concurrent/TestParallelismLimitedExecutorService.java
branches/dev-btm/bigdata-bsbm/
branches/dev-btm/bigdata-lubm/
branches/dev-btm/bigdata-perf/bsbm/RWStore.properties
branches/dev-btm/bigdata-perf/bsbm/WORMStore.properties
branches/dev-btm/bigdata-perf/bsbm/build.properties
branches/dev-btm/bigdata-perf/bsbm/build.xml
branches/dev-btm/bigdata-perf/bsbm/lib/
branches/dev-btm/bigdata-perf/bsbm/lib/jdom.jar
branches/dev-btm/bigdata-perf/bsbm/lib/log4j-1.2.12.jar
branches/dev-btm/bigdata-perf/bsbm/lib/ssj.jar
branches/dev-btm/bigdata-perf/bsbm/src/
branches/dev-btm/bigdata-perf/bsbm/src/java/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/DateGenerator.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/Generator.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/NormalDistGenerator.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/NormalDistRangeGenerator.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/ParetoDistGenerator.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/RandomBucket.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/TextGenerator.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/ValueGenerator.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/BSBMResource.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/Offer.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/Person.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/Producer.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/Product.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/ProductFeature.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/ProductType.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/RatingSite.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/Review.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/Vendor.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/qualification/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/qualification/Qualification.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/qualification/QualificationDefaultValues.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/qualification/QueryResult.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/NTriples.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/ObjectBundle.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/SQLSerializer.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/Serializer.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/TriG.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/Turtle.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/VirtSerializer.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/XMLSerializer.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/AbstractParameterPool.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/ClientManager.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/ClientThread.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/CompiledQuery.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/CompiledQueryMix.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/LocalSPARQLParameterPool.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/NetQuery.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/PreCalcParameterPool.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/Query.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/QueryMix.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/SPARQLConnection.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/SQLConnection.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/SQLParameterPool.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/ServerConnection.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/TestDriver.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/TestDriverDefaultValues.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/tools/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/tools/ResultTransform.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/BSBM.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/DC.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/FOAF.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/ISO3166.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/RDF.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/RDFS.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/REV.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/XSD.java
branches/dev-btm/bigdata-perf/bsbm/src/resources/
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/givennames.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/ignoreQueries.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query1.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query10.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query10desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query10valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query11.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query11desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query11valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query12.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query12desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query12valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query1desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query1valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query2.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query2desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query2valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query3.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query3desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query3valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query4-original.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query4-rewritten.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query4.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query4desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query4valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query5.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query5desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query5valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query6.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query6desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query6valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query7.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query7desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query7valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query8.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query8desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query8valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query9-original.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query9.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query9desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query9valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/querymix.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/titlewords.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/logging/
branches/dev-btm/bigdata-perf/bsbm/src/resources/logging/log4j.properties
branches/dev-btm/bigdata-perf/bsbm/src/test/
branches/dev-btm/bigdata-perf/bsbm/src/test/benchmark/
branches/dev-btm/bigdata-perf/bsbm/src/test/benchmark/bigdata/
branches/dev-btm/bigdata-perf/bsbm/src/test/benchmark/bigdata/TestBSBM.java
branches/dev-btm/bigdata-perf/lubm/LEGAL/
branches/dev-btm/bigdata-perf/lubm/LEGAL/LICENSE.txt
branches/dev-btm/bigdata-perf/lubm/README.txt
branches/dev-btm/bigdata-perf/lubm/RWStore.properties
branches/dev-btm/bigdata-perf/lubm/WORMStore.properties
branches/dev-btm/bigdata-perf/lubm/build.properties
branches/dev-btm/bigdata-perf/lubm/build.xml
branches/dev-btm/bigdata-perf/lubm/lib/
branches/dev-btm/bigdata-perf/lubm/src/
branches/dev-btm/bigdata-perf/lubm/src/java/
branches/dev-btm/bigdata-perf/lubm/src/java/edu/
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/uba/
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/uba/CompressEnum.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/uba/DamlWriter.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/uba/Generator.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/uba/OwlWriter.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/uba/RdfWriter.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/uba/Writer.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/uba/readme.txt
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/ConfigParser.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/KbConfigParser.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/KbSpecification.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/QueryConfigParser.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/QuerySpecification.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/QueryTestResult.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/RepositoryCreator.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/Test.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/api/
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/api/Atom.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/api/Query.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/api/QueryResult.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/api/Repository.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/api/RepositoryFactory.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/bigdata/
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/bigdata/SparqlRepository.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/bigdata/SparqlRepositoryFactory.java
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/config.kb.example.dldb
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/config.kb.example.sesame
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/config.query.example.dldb
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/config.query.example.sesame
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/readme.txt
branches/dev-btm/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/univ-bench.owl
branches/dev-btm/bigdata-perf/lubm/src/resources/
branches/dev-btm/bigdata-perf/lubm/src/resources/answers (U1)/
branches/dev-btm/bigdata-perf/lubm/src/resources/answers (U1)/answers_query14.txt
branches/dev-btm/bigdata-perf/lubm/src/resources/answers (U1)/answers_query6.txt
branches/dev-btm/bigdata-perf/lubm/src/resources/answers (U1)/answers_query8.txt
branches/dev-btm/bigdata-perf/lubm/src/resources/answers (U1)/reference query answers.url
branches/dev-btm/bigdata-perf/lubm/src/resources/config/
branches/dev-btm/bigdata-perf/lubm/src/resources/config/config.kb.sparql
branches/dev-btm/bigdata-perf/lubm/src/resources/config/config.query.sparql
branches/dev-btm/bigdata-perf/l...
[truncated message content] |
|
From: <btm...@us...> - 2010-07-29 17:30:33
|
Revision: 3350
http://bigdata.svn.sourceforge.net/bigdata/?rev=3350&view=rev
Author: btmurphy
Date: 2010-07-29 17:30:26 +0000 (Thu, 29 Jul 2010)
Log Message:
-----------
[branch dev-btm]: modified browser and disco-tool to fallback to a reasonable default nic on failure
Modified Paths:
--------------
branches/dev-btm/bigdata-jini/src/java/com/bigdata/disco/config/disco.config
branches/dev-btm/src/resources/bin/config/browser.config
branches/dev-btm/src/resources/bin/config/serviceStarter.config
branches/dev-btm/src/resources/bin/disco-tool
branches/dev-btm/src/resources/bin/pstart
branches/dev-btm/src/resources/config/jini/reggie.config
branches/dev-btm/src/resources/config/jini/zookeeper.config
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/disco/config/disco.config
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/disco/config/disco.config 2010-07-29 16:54:30 UTC (rev 3349)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/disco/config/disco.config 2010-07-29 17:30:26 UTC (rev 3350)
@@ -18,8 +18,8 @@
com.bigdata.disco.DiscoveryTool {
- private static exportIpAddr =
- NicUtil.getIpAddress("${exportNic}", 0, "${exportHost}");
+ private static exportIpAddr =
+ NicUtil.getIpAddress("default.nic", "default", true);
lookupLocatorConstraints = null;
Modified: branches/dev-btm/src/resources/bin/config/browser.config
===================================================================
--- branches/dev-btm/src/resources/bin/config/browser.config 2010-07-29 16:54:30 UTC (rev 3349)
+++ branches/dev-btm/src/resources/bin/config/browser.config 2010-07-29 17:30:26 UTC (rev 3350)
@@ -9,12 +9,15 @@
import com.bigdata.util.config.NicUtil;
com.sun.jini.example.browser {
- private static exportIpAddr = NicUtil.getIpAddress("${default.nic}",0,"${exportHost}",true);
+
+ private static exportIpAddr =
+ NicUtil.getIpAddress("default.nic", "default", true);
+
private static serverILFactory =
new BasicILFactory(
new BasicMethodConstraints(
new InvocationConstraints(
- new ConnectionRelativeTime(10000), null)),
+ new ConnectionRelativeTime(10000L), null)),
null);
listenerExporter = new BasicJeriExporter(TcpServerEndpoint.getInstance(exportIpAddr,0),
Modified: branches/dev-btm/src/resources/bin/config/serviceStarter.config
===================================================================
--- branches/dev-btm/src/resources/bin/config/serviceStarter.config 2010-07-29 16:54:30 UTC (rev 3349)
+++ branches/dev-btm/src/resources/bin/config/serviceStarter.config 2010-07-29 17:30:26 UTC (rev 3350)
@@ -4,7 +4,9 @@
* needs to start any of the Bigdata services, as
* well as the Jini lookup service and browser.
*/
+import com.bigdata.util.config.ConfigDeployUtil;
import com.bigdata.util.config.ConfigurationUtil;
+import com.bigdata.util.config.NicUtil;
import com.sun.jini.config.ConfigUtil;
import com.sun.jini.start.NonActivatableServiceDescriptor;
@@ -13,16 +15,21 @@
com.sun.jini.start {
+ private static codebaseHost =
+ NicUtil.getIpAddress("bigdata.codebase.host", ConfigDeployUtil.getString("node.serviceNetwork"), false);
+ private static codebasePort =
+ Integer.parseInt( System.getProperty("bigdata.codebase.port", "0") );
+ private static codebaseRootDir =
+ System.getProperty("bigdata.codebase.rootdDir", "." );
+
private static jskCodebase =
ConfigurationUtil.computeCodebase
- ( "${bigdata.codebase.host}",
- "jsk-dl.jar", Integer.parseInt("${bigdata.codebase.port}"),
- "${bigdata.codebase.rootDir}", "none" );
+ ( codebaseHost, "jsk-dl.jar",
+ codebasePort, codebaseRootDir, "none" );
private static servicePolicyFile =
"${appHome}${/}var${/}config${/}policy${/}service.policy";
-
// For starting HTTP codebase class server
private static httpdCodebase = "";
private static httpdPolicyFile = servicePolicyFile;
@@ -92,8 +99,7 @@
// For starting a shard (data) service
- private static shardCodebase = jskCodebase;
-
+ private static shardCodebase = jskCodebase;
private static shardClasspath = "${appHome}${/}lib${/}bigdata.jar";
private static shardImplName = "com.bigdata.shard.ServiceImpl";
private static shardConfig = "${appHome}${/}var${/}config${/}jini${/}shard.config";
@@ -198,9 +204,8 @@
// For starting a lookup service
private static reggieServerCodebase =
ConfigurationUtil.computeCodebase
- ( "${bigdata.codebase.host}",
- "reggie-dl.jar", Integer.parseInt("${bigdata.codebase.port}"),
- "${bigdata.codebase.rootDir}", "none" );
+ ( codebaseHost, "reggie-dl.jar",
+ codebasePort, codebaseRootDir, "none" );
private static reggieCodebase =
ConfigUtil.concat( new String[] { reggieServerCodebase, " ", jskCodebase } );
@@ -226,9 +231,8 @@
// For starting a Jini browser
private static browserServerCodebase =
ConfigurationUtil.computeCodebase
- ( "${bigdata.codebase.host}",
- "browser-dl.jar", Integer.parseInt("${bigdata.codebase.port}"),
- "${bigdata.codebase.rootDir}", "none" );
+ ( codebaseHost, "browser-dl.jar",
+ codebasePort, codebaseRootDir, "none" );
private static browserCodebase =
ConfigUtil.concat( new String[] { browserServerCodebase, " ", jskCodebase } );
Modified: branches/dev-btm/src/resources/bin/disco-tool
===================================================================
--- branches/dev-btm/src/resources/bin/disco-tool 2010-07-29 16:54:30 UTC (rev 3349)
+++ branches/dev-btm/src/resources/bin/disco-tool 2010-07-29 17:30:26 UTC (rev 3350)
@@ -71,16 +71,22 @@
Prints this help message.
Useful properties:
- exportNic=<interfacename>
+ default.nic=<interfacename>
Specifies the name of the network interface on which the
ServiceDiscoveryManager's remote event listener will be exported. This
- property takes precedence over entityExportHost. The default value is
- "eth0".
+ property takes precedence over exportHost. The default behavior
+ regarding this property (in conjunction with the use of NicUtil
+ in the disco.config configuration file to retrieve the interface's
+ associated ip address) is to direct the configuration file to use
+ the IPv4 address of the first active network inteface that can
+ be found on the system.
exportHost=<ipaddress|hostname>
- Specifies the IP address on which the ServiceDiscoveryManager's remote
- event listener will be exported. This property is not used unless
- exportNic is "".
+ Specifies the IP address on which the ServiceDiscoveryManager's
+ remote event listener will be exported. This property will be
+ employed only when the value of the default.nic property
+ is set to the empty string ("") or a value that does not
+ correspond to any of the network interfaces on the system.
Examples:
Show information about all services, discovered through all
@@ -158,7 +164,7 @@
java_props = {
"java.security.manager": "",
"java.net.preferIPv4Stack": "true",
- "exportNic": "eth0",
+ "default.nic": "${default.nic}",
"networkInterface": "all",
"exportHost": socket.gethostname() }
Modified: branches/dev-btm/src/resources/bin/pstart
===================================================================
--- branches/dev-btm/src/resources/bin/pstart 2010-07-29 16:54:30 UTC (rev 3349)
+++ branches/dev-btm/src/resources/bin/pstart 2010-07-29 17:30:26 UTC (rev 3350)
@@ -49,8 +49,12 @@
default.nic=<interfacename>
Specifies the name of the network interface to use by default
for service export and remote communication. This property
- takes precedence over exportHost. The default value for this
- property is "eth0".
+ takes precedence over exportHost. The default behavior
+ regarding this property (in conjunction with configuration
+ files that use NicUtil to retrieve the interface's associated
+ ip address) is to direct the configuration file to use the
+ IPv4 address of the first active network inteface that can
+ be found on the system.
exportHost=<ipaddress|hostname>
Specifies the IP address or host name to use when exporting
@@ -63,11 +67,15 @@
Specifies the network address of the codebase HTTP server. If the
value is an interface name, the IP address assigned to that interface
will be used. If the value is an IP address or hostname, that value
- will be used directly. The default value is "eth0".
+ will be used directly. The default behavior regarding this property
+ (in conjunction with configuration files that use NicUtil to
+ retrieve the interface's associated ip address) is to direct the
+ configuration file to use the IPv4 address of the first active
+ network inteface that can be found on the system.
bigdata.codebase.port=<n>
The port number on <bigdata.codebase.host> on which the HTTP class
- server is running.
+ server is listening.
"""
serviceNames = bigdataServiceMap.keys()
@@ -131,10 +139,10 @@
self.properties['java.security.debug'] = "off"
def setEntityProperties(self):
- self.properties['default.nic'] = "eth0"
+ self.properties['default.nic'] = "${default.nic}"
self.properties['exportHost'] = socket.gethostname()
self.properties['exportPort'] = "0"
- self.properties['bigdata.codebase.host'] = "eth0"
+ self.properties['bigdata.codebase.host'] = "${bigdata.codebase.host}"
self.properties['bigdata.codebase.port'] = "8081"
self.properties['log4j.configuration'] = os.path.join( \
@@ -461,8 +469,6 @@
if params.appHome == "NOT_SET":
params.appHome = dirname(dirname( os.path.abspath(sys.argv[0])) )
- # BTM - params.appHome = dirname(dirname( os.path.abspath(sys.argv[0])) )
-
# print "appHome=" + params.appHome
# Instiantate the object for the service
Modified: branches/dev-btm/src/resources/config/jini/reggie.config
===================================================================
--- branches/dev-btm/src/resources/config/jini/reggie.config 2010-07-29 16:54:30 UTC (rev 3349)
+++ branches/dev-btm/src/resources/config/jini/reggie.config 2010-07-29 17:30:26 UTC (rev 3350)
@@ -17,7 +17,6 @@
private static exportIpAddr =
NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
-
private static exportPort =
Integer.parseInt( System.getProperty("exportPort", "0") );
Modified: branches/dev-btm/src/resources/config/jini/zookeeper.config
===================================================================
--- branches/dev-btm/src/resources/config/jini/zookeeper.config 2010-07-29 16:54:30 UTC (rev 3349)
+++ branches/dev-btm/src/resources/config/jini/zookeeper.config 2010-07-29 17:30:26 UTC (rev 3350)
@@ -7,7 +7,6 @@
* not yet been released as part of the Hadoop Zookeeper
* codebase.
*/
-
import java.net.NetworkInterface;
import com.sun.jini.config.ConfigUtil;
@@ -20,20 +19,18 @@
import net.jini.core.discovery.LookupLocator;
import net.jini.discovery.LookupDiscoveryManager;
-import com.bigdata.util.config.NicUtil;
import com.bigdata.util.config.ConfigDeployUtil;
+import com.bigdata.util.config.NicUtil;
org.apache.zookeeper.server.quorum {
- private static exportNic =
- System.getProperty("exportNic",
- ConfigDeployUtil.getString("node.serviceNetwork"));
- private static exportIpAddr = NicUtil.getIpAddress(exportNic);
+ private static exportIpAddr =
+ NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
private static exportPort =
Integer.parseInt( System.getProperty("exportPort", "0") );
- groupsToJoin = ConfigDeployUtil.getGroupsToDiscover();
- locatorsToJoin = ConfigDeployUtil.getLocatorsToDiscover();
+ private static groupsToJoin = ConfigDeployUtil.getGroupsToDiscover();
+ private static locatorsToJoin = ConfigDeployUtil.getLocatorsToDiscover();
private static exporterTcpServerEndpoint =
TcpServerEndpoint.getInstance(exportIpAddr, exportPort);
@@ -41,9 +38,7 @@
new BasicILFactory(
new BasicMethodConstraints(
new InvocationConstraints(
- new ConnectionRelativeTime(
- ConfigDeployUtil.getLong(
- "rmi.connectTimeout")),
+ new ConnectionRelativeTime( ConfigDeployUtil.getLong("rmi.connectTimeout")),
null)),
null);
@@ -82,7 +77,7 @@
net.jini.discovery.LookupDiscovery {
multicastRequestHost = org.apache.zookeeper.server.quorum.exportIpAddr;
multicastInterfaces = new NetworkInterface[] {
- NicUtil.getNetworkInterface(org.apache.zookeeper.server.quorum.exportNic)
+ NicUtil.getNetworkInterface(org.apache.zookeeper.server.quorum.exportIpAddr)
};
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <btm...@us...> - 2010-07-29 20:31:20
|
Revision: 3363
http://bigdata.svn.sourceforge.net/bigdata/?rev=3363&view=rev
Author: btmurphy
Date: 2010-07-29 20:31:11 +0000 (Thu, 29 Jul 2010)
Log Message:
-----------
merge -r:3339:HEAD(3358) ~/trunk/bigdata ~/bigdata/branches/dev-btm [trunk --> branch dev-btm]
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/btree/BTree.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegment.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentBuilder.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentMultiBlockIterator.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/Options.java
branches/dev-btm/bigdata/src/java/com/bigdata/mdi/LocalPartitionMetadata.java
branches/dev-btm/bigdata/src/java/com/bigdata/mdi/PartitionLocator.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/CompactingMergeTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/IncrementalBuildTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/JoinIndexPartitionTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/MoveTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/OverflowManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/SplitIndexPartitionTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/SplitUtility.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/StoreManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/MetadataService.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexPartitionFencePosts.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexSegmentMultiBlockIterators.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestBuildTask.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestBuildTask2.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestMergeTask.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestOverflow.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestResourceManagerBootstrap.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestSegSplitter.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/DumpFederation.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/TermIdEncoder.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl2.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuads.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuadsAndPipelineJoins.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuadsAndPipelineJoinsWithoutInlining.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithSids.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithSidsWithoutInlining.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithoutSids.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestDescribe.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestOptionals.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestUnions.java
branches/dev-btm/build.properties
Added Paths:
-----------
branches/dev-btm/bigdata/src/releases/RELEASE_0_83_2.txt
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestOrderBy.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestSids.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/sids.rdf
branches/dev-btm/lgpl-utils/LEGAL/lgpl-utils-license.txt
Removed Paths:
-------------
branches/dev-btm/dsi-utils/LEGAL/LICENSE.txt
branches/dev-btm/lgpl-utils/LEGAL/LICENSE.txt
Property Changed:
----------------
branches/dev-btm/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/attr/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/disco/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/
branches/dev-btm/bigdata-perf/
branches/dev-btm/bigdata-perf/lubm/lib/
branches/dev-btm/bigdata-perf/lubm/src/resources/
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util/
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/
branches/dev-btm/dsi-utils/LEGAL/
branches/dev-btm/dsi-utils/lib/
branches/dev-btm/dsi-utils/src/
branches/dev-btm/dsi-utils/src/test/
branches/dev-btm/dsi-utils/src/test/it/
branches/dev-btm/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/osgi/
Property changes on: branches/dev-btm
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/LEXICON_REFACTOR_BRANCH:2633-3304
/branches/bugfix-btm:2594-3237
/branches/fko:3150-3194
/trunk:2575-2594,2596-2877,2882-2903,2910-3339
+ /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/LEXICON_REFACTOR_BRANCH:2633-3304
/branches/bugfix-btm:2594-3237
/branches/fko:3150-3194
/trunk:2575-2594,2596-2877,2882-2903,2910-3358
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/BTree.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/BTree.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/BTree.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -1339,8 +1339,8 @@
oldPmd.getLeftSeparatorKey(), //
oldPmd.getRightSeparatorKey(),//
newResources,//
- oldPmd.getIndexPartitionCause(),//
- "" // history is deprecated.
+ oldPmd.getIndexPartitionCause()//
+// "" // history is deprecated.
);
// update the local partition metadata on our cloned IndexMetadata.
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegment.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegment.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegment.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -623,7 +623,7 @@
*
* @throws IllegalArgumentException
* if the <i>key</i> is <code>null</code>.
- * @throws RUntimeException
+ * @throws RuntimeException
* if the key does not lie within the optional key-range
* constraints for an index partition.
*/
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentBuilder.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentBuilder.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentBuilder.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -1227,9 +1227,9 @@
pmd.getLeftSeparatorKey(),//
pmd.getRightSeparatorKey(),//
null, // No resource metadata for indexSegment.
- pmd.getIndexPartitionCause(),
- pmd.getHistory()+
- "build("+pmd.getPartitionId()+",compactingMerge="+compactingMerge+") "
+ pmd.getIndexPartitionCause()
+// ,pmd.getHistory()+
+// "build("+pmd.getPartitionId()+",compactingMerge="+compactingMerge+") "
)
);
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentMultiBlockIterator.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentMultiBlockIterator.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentMultiBlockIterator.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -151,6 +151,16 @@
*/
private int blockLength = 0;
+ /*
+ * Counters
+ */
+
+ /** The #of leaves read so far. */
+ private long leafReadCount = 0;
+
+ /** The #of blocks read so far. */
+ private long blockReadCount = 0;
+
/**
*
* @param seg
@@ -209,11 +219,25 @@
: seg.findLeafAddr(toKey));
if (pool.getBufferCapacity() < store.getCheckpoint().maxNodeOrLeafLength) {
+
/*
- * Leaves are invariably larger than nodes. If the buffers in the
- * pool are too small to hold the largest record in the index
- * segment then you can not use this iterator.
+ * If the buffers in the pool are too small to hold the largest
+ * record in the index segment then you can not use this iterator.
+ *
+ * Note: We presume that the largest record is therefore a leaf. In
+ * practice this will nearly always be true as nodes have relatively
+ * little metadata per tuple while leaves store the value associated
+ * with the tuple.
+ *
+ * Note: AbstractBTree checks for this condition before choosing
+ * this iterator.
*/
+
+ throw new UnsupportedOperationException(
+ "Record is larger than buffer: maxNodeOrLeafLength="
+ + store.getCheckpoint().maxNodeOrLeafLength
+ + ", bufferCapacity=" + pool.getBufferCapacity());
+
}
if (firstLeafAddr == 0L) {
@@ -345,7 +369,7 @@
throw new IllegalStateException();
if (currentLeaf == null) {
if (log.isTraceEnabled())
- log.trace("Reading first leaf");
+ log.trace("Reading initial leaf");
// acquire the buffer from the pool.
acquireBuffer();
// Read the first block.
@@ -355,6 +379,12 @@
// Return the first leaf.
return leaf;
}
+ if (currentLeaf.identity == lastLeafAddr) {
+ // No more leaves.
+ if (log.isTraceEnabled())
+ log.trace("No more leaves (end of key range)");
+ return null;
+ }
/*
* We need to return the next leaf. We get the address of the next leaf
* from the nextAddr field of the current leaf.
@@ -363,7 +393,7 @@
if (nextLeafAddr == 0L) {
// No more leaves.
if (log.isTraceEnabled())
- log.trace("No more leaves");
+ log.trace("No more leaves (end of segment)");
return null;
}
/*
@@ -411,20 +441,25 @@
throw new IllegalArgumentException();
// offset into the buffer.
- final int toff = (int)(offset - blockOffset);
+ final int offsetWithinBuffer = (int)(offset - blockOffset);
- if (log.isTraceEnabled())
- log.trace("addr=" + addr + "(" + store.toString(addr)
- + "), blockOffset=" + blockOffset+" toff="+toff);
-
// read only view of the leaf in the buffer.
final ByteBuffer tmp = buffer.asReadOnlyBuffer();
- tmp.limit(toff + nbytes);
- tmp.position(toff);
+ tmp.limit(offsetWithinBuffer + nbytes);
+ tmp.position(offsetWithinBuffer);
// decode byte[] as ILeafData.
final ILeafData data = (ILeafData) seg.nodeSer.decode(tmp);
-
+
+ leafReadCount++;
+
+ if (log.isTraceEnabled())
+ log
+ .trace("read leaf: leafReadCount=" + leafReadCount
+ + ", addr=" + addr + "(" + store.toString(addr)
+ + "), blockOffset=" + blockOffset
+ + " offsetWithinBuffer=" + offsetWithinBuffer);
+
// return as Leaf.
return new ImmutableLeaf(seg, addr, data);
@@ -470,6 +505,14 @@
// the #of bytes that we will actually read.
final int nbytes = (int) Math.min(lastOffset - startOffset, b
.capacity());
+ if(log.isTraceEnabled())
+ log.trace("leafAddr=" + store.toString(leafAddr) + ", startOffset="
+ + startOffset + ", lastOffset=" + lastOffset + ", nbytes="
+ + nbytes);
+ if (nbytes == 0) {
+ throw new AssertionError("nbytes=0 : leafAddr"
+ + store.toString(leafAddr) + " : " + this);
+ }
// set the position to zero.
b.position(0);
// set the limit to the #of bytes to be read.
@@ -483,9 +526,29 @@
// update the offset/length in the store for the in memory block
blockOffset = startOffset;
blockLength = nbytes;
+ blockReadCount++;
if (log.isTraceEnabled())
- log.trace("leafAddr=" + leafAddr + ", blockOffset=" + blockOffset
- + ", blockLength=" + blockLength);
+ log.trace("read block: blockReadCount=" + blockReadCount
+ + ", leafAddr=" + store.toString(leafAddr)
+ + ", blockOffset=" + blockOffset + ", blockLength="
+ + blockLength);
}
+ public String toString() {
+ return super.toString() + //
+ "{file=" + store.getFile() + //
+ ",checkpoint="+store.getCheckpoint()+//
+ ",fromKey="+BytesUtil.toString(fromKey)+//
+ ",toKey="+BytesUtil.toString(toKey)+//
+ ",firstLeafAddr=" + store.toString(firstLeafAddr) + //
+ ",lastLeafAddr=" + store.toString(lastLeafAddr) + //
+ ",currentLeaf=" + (currentLeaf!=null?store.toString(currentLeaf.identity):"N/A") + //
+ ",blockOffset="+blockOffset+//
+ ",blockLength="+blockLength+//
+ ",bufferCapacity="+pool.getBufferCapacity()+//
+ ",leafReadCount="+leafReadCount+//
+ ",blockReadCount="+blockReadCount+//
+ "}";
+ }
+
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -470,6 +470,24 @@
}
}
+
+ /**
+ * Need to override commit to ensure the writeCache is flushed prior to
+ * writing the root block.
+ *
+ * For the DiskOnlyStrategy flushing the writeCache also ensures the backing
+ * file is created if the file is temporary.
+ *
+ * Note that the internal call to flush the writeCache must be synchronized
+ * or concurrent writers to the cache will cause problems.
+ */
+ public void commit() {
+ if (writeCache != null) {
+ synchronized(this) {
+ flushWriteCache();
+ }
+ }
+ }
/**
* Writes the {@link #writeCache} through to the disk and its position is
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/Options.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/Options.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/Options.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -402,28 +402,23 @@
String FORCE_ON_COMMIT = AbstractJournal.class.getName()+".forceOnCommit";
/**
- * This boolean option causes application data to be forced to stable
- * storage <em>before</em> we update the root blocks. This option seeks to
- * guarantee that the application data is stable on the disk before the
- * atomic commit. Some operating systems and/or file systems may otherwise
- * choose an ordered write or otherwise process the writes in a different
- * order. This could have the consequence that the root blocks are laid down
- * on the disk before the application data. In this situation a hard failure
- * during the write could result in the loss of application data since the
- * updated root blocks represent the atomic commit point but not all
- * application data was successfully made stable on disk.
+ * This boolean option may be used to request that application data are
+ * forced to stable storage <em>before</em> we update the root blocks
+ * (default {@value #DEFAULT_DOUBLE_SYNC}). This is accomplished by invoking
+ * {@link FileChannel#force(boolean)} before root blocks are updated as part
+ * of the atomic commit protocol in an attempt to guard against operating
+ * systems and/or file systems which may otherwise reorders writes with the
+ * consequence that the root blocks are laid down on the disk before the
+ * application data. In this situation a hard failure during the root block
+ * write could result in the loss of application data since the updated root
+ * blocks represent the atomic commit point but not all application data was
+ * successfully made stable on disk. However, note that there are often
+ * multiple cache layers in use by the operating system, the disk
+ * controller, and the disk. Therefore durability is thus best achieved
+ * through a mixture of methods, which can include battery powered hardware
+ * write cache and/or replication.
*
- * @deprecated This option does NOT provide a sufficient guarantee when a
- * write cache is in use by the operating system or the disk if
- * the layered write caches return before all data is safely on
- * disk (or in a battery powered cache). In order to protect
- * against this you MUST disable the write cache layers in the
- * operating system and the disk drive such that
- * {@link FileChannel#force(boolean)} will not return until the
- * data are in fact on stable storage. If you disable the OS and
- * disk write cache then you do NOT need to specify this option
- * since writes will be ordered and all data will be on disk
- * before we update the commit blocks.
+ * @see #DEFAULT_DOUBLE_SYNC
*/
String DOUBLE_SYNC = AbstractJournal.class.getName()+".doubleSync";
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/mdi/LocalPartitionMetadata.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/mdi/LocalPartitionMetadata.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/mdi/LocalPartitionMetadata.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -41,7 +41,6 @@
import com.bigdata.btree.IndexSegmentStore;
import com.bigdata.journal.Journal;
import com.bigdata.service.DataService;
-import com.bigdata.service.Event;
/**
* An immutable object providing metadata about a local index partition,
@@ -80,6 +79,9 @@
/**
*
* @see #getSourcePartitionId()
+ *
+ * @deprecated MoveTask manages without this field (it was required by the
+ * previous MOVE implementation).
*/
private int sourcePartitionId;
@@ -111,53 +113,53 @@
*/
private IndexPartitionCause cause;
- /**
- * A history of operations giving rise to the current partition metadata.
- * E.g., register(timestamp), copyOnOverflow(timestamp), split(timestamp),
- * join(partitionId,partitionId,timestamp), etc. This is truncated when
- * serialized to keep it from growing without bound.
- *
- * @deprecated See {@link #getHistory()}
- */
- private String history;
+// /**
+// * A history of operations giving rise to the current partition metadata.
+// * E.g., register(timestamp), copyOnOverflow(timestamp), split(timestamp),
+// * join(partitionId,partitionId,timestamp), etc. This is truncated when
+// * serialized to keep it from growing without bound.
+// *
+// * @deprecated See {@link #getHistory()}
+// */
+// private String history;
+//
+// /**
+// * If the history string exceeds {@link #MAX_HISTORY_LENGTH} characters then
+// * truncates it to the last {@link #MAX_HISTORY_LENGTH}-3 characters,
+// * prepends "...", and returns the result. Otherwise returns the entire
+// * history string.
+// *
+// * @deprecated See {@link #history}
+// */
+// protected String getTruncatedHistory() {
+//
+// if (MAX_HISTORY_LENGTH == 0)
+// return "";
+//
+// String history = this.history;
+//
+// if(history.length() > MAX_HISTORY_LENGTH) {
+//
+// /*
+// * Truncate the history.
+// */
+//
+// final int len = history.length();
+//
+// final int fromIndex = len - (MAX_HISTORY_LENGTH - 3);
+//
+// assert fromIndex > 0 : "len=" + len + ", fromIndex=" + fromIndex
+// + ", maxHistoryLength=" + MAX_HISTORY_LENGTH;
+//
+// history = "..." + history.substring(fromIndex, len);
+//
+// }
+//
+// return history;
+//
+// }
/**
- * If the history string exceeds {@link #MAX_HISTORY_LENGTH} characters then
- * truncates it to the last {@link #MAX_HISTORY_LENGTH}-3 characters,
- * prepends "...", and returns the result. Otherwise returns the entire
- * history string.
- *
- * @deprecated See {@link #history}
- */
- protected String getTruncatedHistory() {
-
- if (MAX_HISTORY_LENGTH == 0)
- return "";
-
- String history = this.history;
-
- if(history.length() > MAX_HISTORY_LENGTH) {
-
- /*
- * Truncate the history.
- */
-
- final int len = history.length();
-
- final int fromIndex = len - (MAX_HISTORY_LENGTH - 3);
-
- assert fromIndex > 0 : "len=" + len + ", fromIndex=" + fromIndex
- + ", maxHistoryLength=" + MAX_HISTORY_LENGTH;
-
- history = "..." + history.substring(fromIndex, len);
-
- }
-
- return history;
-
- }
-
- /**
* De-serialization constructor.
*/
public LocalPartitionMetadata() {
@@ -199,21 +201,21 @@
* the remote {@link DataService} will fill it in on arrival.
* @param cause
* The underlying cause for the creation of the index partition.
- * @param history
- * A human interpretable history of the index partition. The
- * history is a series of whitespace delimited records each of
- * more or less the form <code>foo(x,y,z)</code>. The history
- * gets truncated when the {@link LocalPartitionMetadata} is
- * serialized in order to prevent it from growing without bound.
*/
+// * @param history
+// * A human interpretable history of the index partition. The
+// * history is a series of whitespace delimited records each of
+// * more or less the form <code>foo(x,y,z)</code>. The history
+// * gets truncated when the {@link LocalPartitionMetadata} is
+// * serialized in order to prevent it from growing without bound.
public LocalPartitionMetadata(//
final int partitionId,//
final int sourcePartitionId,//
final byte[] leftSeparatorKey,//
final byte[] rightSeparatorKey,//
final IResourceMetadata[] resources,//
- final IndexPartitionCause cause,
- final String history
+ final IndexPartitionCause cause
+// final String history
) {
/*
@@ -232,7 +234,7 @@
this.cause = cause;
- this.history = history;
+// this.history = history;
/*
* Test arguments.
@@ -440,23 +442,23 @@
}
- /**
- * A history of the changes to the index partition.
- *
- * @deprecated I've essentially disabled the history (it is always empty
- * when it is persisted). I found it nearly impossible to read.
- * There are much saner ways to track what is going on in the
- * federation. An analysis of the {@link Event} log is much more
- * useful. If nothing else, you could examine the index
- * partition in the metadata index by scanning the commit points
- * and reading its state in each commit and reporting all state
- * changes.
- */
- final public String getHistory() {
-
- return history;
-
- }
+// /**
+// * A history of the changes to the index partition.
+// *
+// * @deprecated I've essentially disabled the history (it is always empty
+// * when it is persisted). I found it nearly impossible to read.
+// * There are much saner ways to track what is going on in the
+// * federation. An analysis of the {@link Event} log is much more
+// * useful. If nothing else, you could examine the index
+// * partition in the metadata index by scanning the commit points
+// * and reading its state in each commit and reporting all state
+// * changes.
+// */
+// final public String getHistory() {
+//
+// return history;
+//
+// }
final public int hashCode() {
@@ -466,7 +468,7 @@
}
// Note: used by assertEquals in the test cases.
- public boolean equals(Object o) {
+ public boolean equals(final Object o) {
if (this == o)
return true;
@@ -520,7 +522,7 @@
", rightSeparator="+BytesUtil.toString(rightSeparatorKey)+
", resourceMetadata="+Arrays.toString(resources)+
", cause="+cause+
- ", history="+history+
+// ", history="+history+
"}"
;
@@ -537,6 +539,17 @@
* but that field is only serialized for a journal.
*/
private static final transient short VERSION1 = 0x1;
+
+ /**
+ * This version serializes the {@link #partitionId} as 32-bits clean and
+ * gets rid of the <code>history</code> field.
+ */
+ private static final transient short VERSION2 = 0x2;
+
+ /**
+ * The current version.
+ */
+ private static final transient short VERSION = VERSION2;
public void readExternal(final ObjectInput in) throws IOException,
ClassNotFoundException {
@@ -546,13 +559,18 @@
switch (version) {
case VERSION0:
case VERSION1:
+ case VERSION2:
break;
default:
throw new IOException("Unknown version: " + version);
}
-
- partitionId = (int) LongPacker.unpackLong(in);
+ if (version < VERSION2) {
+ partitionId = (int) LongPacker.unpackLong(in);
+ } else {
+ partitionId = in.readInt();
+ }
+
sourcePartitionId = in.readInt(); // MAY be -1.
final int nresources = ShortPacker.unpackShort(in);
@@ -579,7 +597,9 @@
cause = (IndexPartitionCause)in.readObject();
- history = in.readUTF();
+ if (version < VERSION2) {
+ /* history = */in.readUTF();
+ }
resources = nresources>0 ? new IResourceMetadata[nresources] : null;
@@ -613,9 +633,13 @@
public void writeExternal(final ObjectOutput out) throws IOException {
- ShortPacker.packShort(out, VERSION1);
+ ShortPacker.packShort(out, VERSION);
- LongPacker.packLong(out, partitionId);
+ if (VERSION < VERSION2) {
+ LongPacker.packLong(out, partitionId);
+ } else {
+ out.writeInt(partitionId);
+ }
out.writeInt(sourcePartitionId); // MAY be -1.
@@ -640,7 +664,9 @@
out.writeObject(cause);
- out.writeUTF(getTruncatedHistory());
+ if (VERSION < VERSION2) {
+ out.writeUTF("");// getTruncatedHistory()
+ }
/*
* Note: we serialize using the IResourceMetadata interface so that we
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/mdi/PartitionLocator.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/mdi/PartitionLocator.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/mdi/PartitionLocator.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -224,19 +224,36 @@
}
+ /**
+ * The original version.
+ */
private static final transient short VERSION0 = 0x0;
+
+ /**
+ * The {@link #partitionId} is now 32-bits clean.
+ */
+ private static final transient short VERSION1 = 0x0;
+ /**
+ * The current version.
+ */
+ private static final transient short VERSION = VERSION1;
+
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
final short version = ShortPacker.unpackShort(in);
- if (version != VERSION0) {
-
+ if (version != VERSION0 && version != VERSION1) {
+
throw new IOException("Unknown version: "+version);
}
- partitionId = (int)LongPacker.unpackLong(in);
+ if (version < VERSION1) {
+ partitionId = (int) LongPacker.unpackLong(in);
+ } else {
+ partitionId = in.readInt();
+ }
dataServiceUUID = new UUID(in.readLong()/*MSB*/,in.readLong()/*LSB*/);
@@ -264,9 +281,13 @@
public void writeExternal(ObjectOutput out) throws IOException {
- ShortPacker.packShort(out, VERSION0);
+ ShortPacker.packShort(out, VERSION);
- LongPacker.packLong(out, partitionId);
+ if (VERSION < VERSION1) {
+ LongPacker.packLong(out, partitionId);
+ } else {
+ out.writeInt(partitionId);
+ }
out.writeLong(dataServiceUUID.getMostSignificantBits());
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/resources/CompactingMergeTask.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/resources/CompactingMergeTask.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/resources/CompactingMergeTask.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -1188,21 +1188,22 @@
currentpmd.getLeftSeparatorKey(),//
currentpmd.getRightSeparatorKey(),//
newResources, //
- currentpmd.getIndexPartitionCause(),
- currentpmd.getHistory()
- + OverflowActionEnum.Merge//
- + "(lastCommitTime="
- + segmentMetadata.getCreateTime()//
- + ",btreeEntryCount="
- + btree.getEntryCount()//
- + ",segmentEntryCount="
- + buildResult.builder.getCheckpoint().nentries//
- + ",segment="
- + segmentMetadata.getUUID()//
- + ",counter="
- + btree.getCounter().get()//
- + ",oldResources="
- + Arrays.toString(currentResources) + ") "));
+ currentpmd.getIndexPartitionCause()
+// currentpmd.getHistory()
+// + OverflowActionEnum.Merge//
+// + "(lastCommitTime="
+// + segmentMetadata.getCreateTime()//
+// + ",btreeEntryCount="
+// + btree.getEntryCount()//
+// + ",segmentEntryCount="
+// + buildResult.builder.getCheckpoint().nentries//
+// + ",segment="
+// + segmentMetadata.getUUID()//
+// + ",counter="
+// + btree.getCounter().get()//
+// + ",oldResources="
+// + Arrays.toString(currentResources) + ") "
+ ));
// update the metadata associated with the btree
btree.setIndexMetadata(indexMetadata);
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/resources/IncrementalBuildTask.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/resources/IncrementalBuildTask.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/resources/IncrementalBuildTask.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -552,21 +552,22 @@
currentpmd.getLeftSeparatorKey(),//
currentpmd.getRightSeparatorKey(),//
newResources, //
- currentpmd.getIndexPartitionCause(),
- currentpmd.getHistory()
- + OverflowActionEnum.Build//
- + "(lastCommitTime="
- + segmentMetadata.getCreateTime()//
- + ",segment="
- + segmentMetadata.getUUID()//
- + ",#buildSources="
- + buildResult.sourceCount//
- + ",merge="
- + buildResult.compactingMerge//
- + ",counter="
- + btree.getCounter().get()//
- + ",oldResources="
- + Arrays.toString(currentResources) + ") "));
+ currentpmd.getIndexPartitionCause()
+// , currentpmd.getHistory()
+// + OverflowActionEnum.Build//
+// + "(lastCommitTime="
+// + segmentMetadata.getCreateTime()//
+// + ",segment="
+// + segmentMetadata.getUUID()//
+// + ",#buildSources="
+// + buildResult.sourceCount//
+// + ",merge="
+// + buildResult.compactingMerge//
+// + ",counter="
+// + btree.getCounter().get()//
+// + ",oldResources="
+// + Arrays.toString(currentResources) + ") "
+ ));
// update the metadata associated with the btree
btree.setIndexMetadata(indexMetadata);
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/resources/JoinIndexPartitionTask.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/resources/JoinIndexPartitionTask.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/resources/JoinIndexPartitionTask.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -289,9 +289,10 @@
// Note: the live journal.
getJournal().getResourceMetadata() //
},//
- IndexPartitionCause.join(resourceManager),
- // new history line.
- summary+" "));
+ IndexPartitionCause.join(resourceManager)
+// // new history line.
+// , summary+" "
+ ));
/*
* Set the updated index metadata on the btree (required for it
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/resources/MoveTask.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/resources/MoveTask.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/resources/MoveTask.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -1368,9 +1368,10 @@
// Historical writes from the source DS.
historySegmentMetadata//
},
- IndexPartitionCause.move(resourceManager),
- // history line.
- oldpmd.getHistory() + summary + " "));
+ IndexPartitionCause.move(resourceManager)
+// // history line.
+// ,oldpmd.getHistory() + summary + " "
+ ));
/*
* Create the BTree to aborb writes for the target index
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/resources/OverflowManager.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/resources/OverflowManager.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/resources/OverflowManager.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -2486,16 +2486,18 @@
oldpmd.getLeftSeparatorKey(),//
oldpmd.getRightSeparatorKey(),//
newResources, //
- oldpmd.getIndexPartitionCause(), oldpmd
- .getHistory()
- + OverflowActionEnum.Copy
- + "(lastCommitTime="
- + lastCommitTime
- + ",entryCount="
- + entryCount
- + ",counter="
- + oldBTree.getCounter().get()
- + ") "));
+ oldpmd.getIndexPartitionCause()//
+// , oldpmd
+// .getHistory()
+// + OverflowActionEnum.Copy
+// + "(lastCommitTime="
+// + lastCommitTime
+// + ",entryCount="
+// + entryCount
+// + ",counter="
+// + oldBTree.getCounter().get()
+// + ") "
+ ));
} else {
@@ -2535,15 +2537,17 @@
oldpmd.getLeftSeparatorKey(),//
oldpmd.getRightSeparatorKey(),//
newResources, //
- oldpmd.getIndexPartitionCause(), oldpmd
- .getHistory()
- + "overflow(lastCommitTime="
- + lastCommitTime
- + ",entryCount="
- + entryCount
- + ",counter="
- + oldBTree.getCounter().get()
- + ") "));
+ oldpmd.getIndexPartitionCause()//
+// , oldpmd
+// .getHistory()
+// + "overflow(lastCommitTime="
+// + lastCommitTime
+// + ",entryCount="
+// + entryCount
+// + ",counter="
+// + oldBTree.getCounter().get()
+// + ") "
+ ));
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/resources/SplitIndexPartitionTask.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/resources/SplitIndexPartitionTask.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/resources/SplitIndexPartitionTask.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -844,12 +844,12 @@
resourceManager.getLiveJournal()
.getResourceMetadata(),
splitResult.buildResults[i].segmentMetadata },
- IndexPartitionCause.split(resourceManager),
- /*
- * Note: history is record of the split.
- */
- pmd.getHistory() + summary + " ")//
- );
+ IndexPartitionCause.split(resourceManager)
+// /*
+// * Note: history is record of the split.
+// */
+// , pmd.getHistory() + summary + " "//
+ ));
/*
* create new btree.
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/resources/SplitUtility.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/resources/SplitUtility.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/resources/SplitUtility.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -400,11 +400,12 @@
/*
* Note: cause will be set by the atomic update task.
*/
- null,//
- oldpmd.getHistory()
- + "chooseTailSplitPoint(oldPartitionId="
- + oldpmd.getPartitionId() + ",nsplits=" + 2
- + ",newPartitionId=" + partitionId + ") ");
+ null//
+// , oldpmd.getHistory()
+// + "chooseTailSplitPoint(oldPartitionId="
+// + oldpmd.getPartitionId() + ",nsplits=" + 2
+// + ",newPartitionId=" + partitionId + ") "
+ );
final int fromIndex = 0;
@@ -437,11 +438,12 @@
* Note: Cause will be set by the atomic update for the
* split task.
*/
- null,//
- oldpmd.getHistory()
- + "chooseTailSplitPoint(oldPartitionId="
- + oldpmd.getPartitionId() + ",nsplits=" + 2
- + ",newPartitionId=" + partitionId + ") ");
+ null//
+// , oldpmd.getHistory()
+// + "chooseTailSplitPoint(oldPartitionId="
+// + oldpmd.getPartitionId() + ",nsplits=" + 2
+// + ",newPartitionId=" + partitionId + ") "
+ );
/*
* Note: The index of the last tuple in the btree will be the
@@ -1050,11 +1052,12 @@
/*
* Note: cause will be set by the atomic update task.
*/
- null,//
- oldpmd.getHistory()
- + "chooseSplitPoint(oldPartitionId="
- + oldpmd.getPartitionId() + ",nsplits=" + N
- + ",newPartitionId=" + partitionId + ") ");
+ null //
+// , oldpmd.getHistory()
+// + "chooseSplitPoint(oldPartitionId="
+// + oldpmd.getPartitionId() + ",nsplits=" + N
+// + ",newPartitionId=" + partitionId + ") "
+ );
final Split split = new Split(newpmd, fromIndex, toIndex);
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/resources/StoreManager.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/resources/StoreManager.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/resources/StoreManager.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -2603,14 +2603,15 @@
getResourceMetadata() //
},
// cause
- IndexPartitionCause.register(resourceManager),
- /*
- * Note: Retains whatever history given by the
- * caller.
- */
- pmd.getHistory() + "register(name=" + name
- + ",partitionId="
- + pmd.getPartitionId() + ") "));
+ IndexPartitionCause.register(resourceManager)
+// /*
+// * Note: Retains whatever history given by the
+// * caller.
+// */
+// , pmd.getHistory() + "register(name=" + name
+// + ",partitionId="
+// + pmd.getPartitionId() + ") "
+ ));
} else {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/MetadataService.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/MetadataService.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/MetadataService.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -1123,11 +1123,11 @@
* service.
*/
null, // [resources] Signal to the RegisterIndexTask.
- null, // [cause] Signal to RegisterIndexTask
- /*
- * History.
- */
- "createScaleOutIndex(name="+scaleOutIndexName+") "
+ null // [cause] Signal to RegisterIndexTask
+// /*
+// * History.
+// */
+// ,"createScaleOutIndex(name="+scaleOutIndexName+") "
));
dataServices[i].registerIndex(DataService
Copied: branches/dev-btm/bigdata/src/releases/RELEASE_0_83_2.txt (from rev 3351, trunk/bigdata/src/releases/RELEASE_0_83_2.txt)
===================================================================
--- branches/dev-btm/bigdata/src/releases/RELEASE_0_83_2.txt (rev 0)
+++ branches/dev-btm/bigdata/src/releases/RELEASE_0_83_2.txt 2010-07-29 20:31:11 UTC (rev 3363)
@@ -0,0 +1,65 @@
+This is a bigdata (R) snapshot release. This release is capable of loading 1B
+triples in under one hour on a 15 node cluster and has been used to load up to
+13B triples on the same cluster. JDK 1.6 is required.
+
+See [1] for instructions on installing bigdata(R), [2] for the javadoc and [3]
+and [4] for news, questions, and the latest developments. For more information
+about SYSTAP, LLC and bigdata, see [5].
+
+Please note that we recommend checking out the code from SVN using the tag for
+this release. The code will build automatically under eclipse. You can also
+build the code using the ant script. The cluster installer requires the use of
+the ant script. You can checkout this release from the following URL:
+
+ https://bigdata.svn.sourceforge.net/svnroot/bigdata/branches/BIGDATA_RELEASE_0_83_2
+
+New features:
+
+- This release provides a bug fix for issue#118. Upgrade to this release is
+ advised. See https://sourceforge.net/apps/trac/bigdata/ticket/118 for details.
+
+- Inlining XSD numerics, xsd:boolean, or custom datatype extensions
+ into the statement indices. Inlining provides a smaller footprint
+ and faster queries for data using XSD numeric datatypes. In order
+ to introduce inlining we were forced to make a change in the
+ physical schema for the RDF database which breaks binary
+ compatibility for existing stores. The recommended migration path
+ is to export the data and import it into a new bigdata instance.
+
+- Refactor of the dynamic sharding mechanism for higher performance.
+
+- The SparseRowStore has been modified to make Unicode primary keys
+ decodable by representing Unicode primary keys using UTF8 rather
+ than Unicode sort keys. This change also allows the SparseRowStore
+ to work with the JDK collator option which embeds nul bytes into
+ Unicode sort keys. This change breaks binary compatibility, but
+ there is an option for historical compatibility.
+
+The roadmap for the next releases include:
+
+- Query optimizations;
+
+- Support for high-volume analytic query workloads and SPARQL aggregations;
+
+- High availability for the journal and the cluster;
+
+- Simplified deployment, configuration, and administration for clusters.
+
+For more information, please see the following links:
+
+[1] http://bigdata.wiki.sourceforge.net/GettingStarted
+[2] http://www.bigdata.com/bigdata/docs/api/
+[3] http://sourceforge.net/projects/bigdata/
+[4] http://www.bigdata.com/blog
+[5] http://www.systap.com/bigdata.htm
+
+About bigdata:
+
+Bigdata\xAE is a horizontally-scaled, general purpose storage and computing fabric
+for ordered data (B+Trees), designed to operate on either a single server or a
+cluster of commodity hardware. Bigdata\xAE uses dynamically partitioned key-range
+shards in order to remove any realistic scaling limits - in principle, bigdata\xAE
+may be deployed on 10s, 100s, or even thousands of machines and new capacity may
+be added incrementally without requiring the full reload of all data. The bigdata\xAE
+RDF database supports RDFS and OWL Lite reasoning, high-level query (SPARQL),
+and datum level provenance.
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexPartitionFencePosts.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexPartitionFencePosts.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexPartitionFencePosts.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -78,8 +78,8 @@
new byte[]{}, // leftSeparator
null, // rightSeparator
null, // no resource descriptions.
- null, // no cause.
- "" // history
+ null // no cause.
+// , "" // history
));
BTree ndx = BTree.create(new SimpleMemoryRawStore(),metadata);
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexSegmentMultiBlockIterators.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexSegmentMultiBlockIterators.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexSegmentMultiBlockIterators.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -28,6 +28,7 @@
package com.bigdata.btree;
import java.io.File;
+import java.util.Random;
import java.util.UUID;
import com.bigdata.btree.IndexSegmentBuilder.BuildEnum;
@@ -271,6 +272,8 @@
// verify that the iterator is exhausted.
assertFalse(itr.hasNext());
+ doRandomScanTest(btree, seg, 10/* ntests */);
+
} finally {
seg.getStore().destroy();
@@ -280,6 +283,43 @@
}
/**
+ * Unit test builds an empty index segment and then verifies the behavior of
+ * the {@link IndexSegmentMultiBlockIterator}.
+ *
+ * @throws Exception
+ */
+ public void test_emptyIndexSegment() throws Exception {
+
+ final BTree btree = BTree.createTransient(new IndexMetadata(UUID
+ .randomUUID()));
+
+ final IndexSegmentBuilder builder = TestIndexSegmentBuilderWithLargeTrees
+ .doBuildIndexSegment(getName(), btree, 32/* m */,
+ BuildEnum.TwoPass, bufferNodes);
+
+ final IndexSegment seg = new IndexSegmentStore(builder.outFile)
+ .loadIndexSegment();
+
+ try {
+
+ final IndexSegmentMultiBlockIterator<?> itr = new IndexSegmentMultiBlockIterator(
+ seg, DirectBufferPool.INSTANCE_10M, null/* fromKey */,
+ null/* toKey */, IRangeQuery.DEFAULT);
+
+ assertFalse(itr.hasNext());
+
+ // verify the data.
+ testMultiBlockIterator(btree, seg);
+
+ } finally {
+
+ seg.getStore().destroy();
+
+ }
+
+ }
+
+ /**
* Test build around an {@link IndexSegment} having a default branching
* factor and a bunch of leaves totally more than 1M in size on the disk.
*/
@@ -288,8 +328,13 @@
final BTree btree = BTree.createTransient(new IndexMetadata(UUID
.randomUUID()));
- for (int i = 0; i < 1000000; i++) {
+ final int LIMIT = 1000000;
+
+ // populate the index.
+ for (int i = 0; i < LIMIT; i++) {
+
btree.insert(i, i);
+
}
final IndexSegmentBuilder builder = TestIndexSegmentBuilderWithLargeTrees
@@ -336,6 +381,9 @@
// verify the data.
testMultiBlockIterator(btree, seg);
+
+ // random iterator scan tests.
+ doRandomScanTest(btree, seg, 100/* ntests */);
} finally {
@@ -345,4 +393,112 @@
}
+ /**
+ * Do a bunch of random iterator scans. Each scan will start at a random key
+ * and run to a random key.
+ *
+ * @param groundTruth
+ * The ground truth B+Tree.
+ * @param actual
+ * The index segment built from that B+Tree.
+ * @param ntests
+ * The #of scans to run.
+ */
+ private void doRandomScanTest(final BTree groundTruth,
+ final IndexSegment actual, final int ntests) {
+
+ final Random r = new Random();
+
+ final int n = groundTruth.getEntryCount();
+
+ // point query beyond the last tuple in the index segment.
+ {
+
+ final int fromIndex = n - 1;
+
+ final byte[] fromKey = groundTruth.keyAt(fromIndex);
+
+ final byte[] toKey = BytesUtil.successor(fromKey.clone());
+
+ final ITupleIterator<?> expectedItr = groundTruth
+ .rangeIterator(fromKey, toKey, 0/* capacity */,
+ IRangeQuery.DEFAULT, null/* filter */);
+
+ final IndexSegmentMultiBlockIterator<?> actualItr = new IndexSegmentMultiBlockIterator(
+ actual, DirectBufferPool.INSTANCE_10M, fromKey, toKey,
+ IRangeQuery.DEFAULT);
+
+ assertSameEntryIterator(expectedItr, actualItr);
+
+ }
+
+ // random point queries.
+ for (int i = 0; i < ntests; i++) {
+
+ final int fromIndex = r.nextInt(n);
+
+ final byte[] fromKey = groundTruth.keyAt(fromIndex);
+
+ final byte[] toKey = BytesUtil.successor(fromKey.clone());
+
+ final ITupleIterator<?> expectedItr = groundTruth
+ .rangeIterator(fromKey, toKey, 0/* capacity */,
+ IRangeQuery.DEFAULT, null/* filter */);
+
+ final IndexSegmentMultiBlockIterator<?> actualItr = new IndexSegmentMultiBlockIterator(
+ actual, DirectBufferPool.INSTANCE_10M, fromKey, toKey,
+ IRangeQuery.DEFAULT);
+
+ assertSameEntryIterator(expectedItr, actualItr);
+
+ }
+
+ // random range queries with small range of spanned keys (0 to 10).
+ for (int i = 0; i < ntests; i++) {
+
+ final int fromIndex = r.nextInt(n);
+
+ final byte[] fromKey = groundTruth.keyAt(fromIndex);
+
+ final byte[] toKey = groundTruth.keyAt(Math.min(fromIndex
+ + r.nextInt(10), n - 1));
+
+ final ITupleIterator<?> expectedItr = groundTruth
+ .rangeIterator(fromKey, toKey, 0/* capacity */,
+ IRangeQuery.DEFAULT, null/* filter */);
+
+ final IndexSegmentMultiBlockIterator<?> actualItr = new IndexSegmentMultiBlockIterator(
+ actual, DirectBufferPool.INSTANCE_10M, fromKey, toKey,
+ IRangeQuery.DEFAULT);
+
+ assertSameEntryIterator(expectedItr, actualItr);
+
+ }
+
+ // random range queries with random #of spanned keys.
+ for (int i = 0; i < ntests; i++) {
+
+ final int fromIndex = r.nextInt(n);
+
+ final int toIndex = fromIndex + r.nextInt(n - fromIndex + 1);
+
+ final byte[] fromKey = groundTruth.keyAt(fromIndex);
+
+ final byte[] toKey = toIndex >= n ? null : groundTruth
+ .keyAt(toIndex);
+
+ final ITupleIterator<?> expectedItr = groundTruth
+ .rangeIterator(fromKey, toKey, 0/* capacity */,
+ IRangeQuery.DEFAULT, null/* filter */);
+
+ final IndexSegmentMultiBlockIterator<?> actualItr = new IndexSegmentMultiBlockIterator(
+ actual, DirectBufferPool.INSTANCE_10M, fromKey, toKey,
+ IRangeQuery.DEFAULT);
+
+ assertSameEntryIterator(expectedItr, actualItr);
+
+ }
+
+ }
+
}
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerTestCase.java
===================================================================
--- branches/dev-btm/...
[truncated message content] |
|
From: <btm...@us...> - 2010-07-30 13:51:55
|
Revision: 3372
http://bigdata.svn.sourceforge.net/bigdata/?rev=3372&view=rev
Author: btmurphy
Date: 2010-07-30 13:51:43 +0000 (Fri, 30 Jul 2010)
Log Message:
-----------
merge -r:3358:HEAD(3370) ~/bigdata/trunk ~/bigdata/branches/dev-btm [trunk --> branch dev-btm]
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BlobOverflowHandler.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
branches/dev-btm/bigdata/src/java/com/bigdata/cache/HardReferenceGlobalLRU.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java
branches/dev-btm/bigdata/src/test/com/bigdata/cache/TestAll.java
branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestConcurrentJournal.java
branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestJournalBasics.java
branches/dev-btm/bigdata/src/test/com/bigdata/relation/locator/TestDefaultResourceLocator.java
branches/dev-btm/bigdata/src/test/com/bigdata/search/TestKeyBuilder.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestBasicIndexStuff.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestDistributedTransactionService.java
branches/dev-btm/bigdata/src/test/com/bigdata/sparse/TestKeyEncodeDecode.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithSids.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithSidsWithoutInlining.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataConnectionTest.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataStoreTest.java
Removed Paths:
-------------
branches/dev-btm/CVSROOT/
branches/dev-btm/bigdata-master-pom/
Property Changed:
----------------
branches/dev-btm/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/attr/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/disco/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/
branches/dev-btm/bigdata-perf/
branches/dev-btm/bigdata-perf/lubm/lib/
branches/dev-btm/bigdata-perf/lubm/src/resources/
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util/
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/
branches/dev-btm/dsi-utils/LEGAL/
branches/dev-btm/dsi-utils/lib/
branches/dev-btm/dsi-utils/src/
branches/dev-btm/dsi-utils/src/test/
branches/dev-btm/dsi-utils/src/test/it/
branches/dev-btm/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/osgi/
Property changes on: branches/dev-btm
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/LEXICON_REFACTOR_BRANCH:2633-3304
/branches/bugfix-btm:2594-3237
/branches/fko:3150-3194
/trunk:2575-2594,2596-2877,2882-2903,2910-3358
+ /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/LEXICON_REFACTOR_BRANCH:2633-3304
/branches/bugfix-btm:2594-3237
/branches/fko:3150-3194
/trunk:2575-2594,2596-2877,2882-2903,2910-3370
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BlobOverflowHandler.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BlobOverflowHandler.java 2010-07-30 13:51:36 UTC (rev 3371)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BlobOverflowHandler.java 2010-07-30 13:51:43 UTC (rev 3372)
@@ -6,8 +6,10 @@
import com.bigdata.btree.IOverflowHandler;
import com.bigdata.btree.ITuple;
+import com.bigdata.btree.keys.IKeyBuilder;
import com.bigdata.btree.keys.KeyBuilder;
import com.bigdata.io.DataOutputBuffer;
+import com.bigdata.rawstore.Bytes;
import com.bigdata.rawstore.IBlock;
import com.bigdata.rawstore.IRawStore;
@@ -33,7 +35,7 @@
}
- DataOutputBuffer buf;
+ private transient DataOutputBuffer buf;
public void close() {
@@ -62,6 +64,8 @@
}
+ final IKeyBuilder keyBuilder = new KeyBuilder(Bytes.SIZEOF_LONG);
+
if (addr == 0L) {
/*
@@ -69,7 +73,7 @@
* their address.
*/
- return KeyBuilder.asSortKey(0L);
+ return keyBuilder.append(0L).getKey();
}
@@ -143,7 +147,7 @@
}
// the address of the block on the target store.
- return KeyBuilder.asSortKey(addr2);
+ return keyBuilder.append(addr2).getKey();
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-07-30 13:51:36 UTC (rev 3371)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-07-30 13:51:43 UTC (rev 3372)
@@ -1081,17 +1081,25 @@
* Note: This method is thread-safe.
* <p>
* Note: Strings are Unicode safe for the default locale. See
- * {@link Locale#getDefault()}. If you require a specific local or
- * different locals at different times or for different indices then you
- * MUST provision and apply your own {@link KeyBuilder}.
+ * {@link Locale#getDefault()}. If you require a specific local or different
+ * locals at different times or for different indices then you MUST
+ * provision and apply your own {@link KeyBuilder}.
*
* @param val
* An application key.
*
- * @return The unsigned byte[] equivilent of that key. This will be
- * <code>null</code> iff the <i>key</i> is <code>null</code>.
- * If the <i>key</i> is a byte[], then the byte[] itself will be
- * returned.
+ * @return The unsigned byte[] equivalent of that key. This will be
+ * <code>null</code> iff the <i>key</i> is <code>null</code>. If the
+ * <i>key</i> is a byte[], then the byte[] itself will be returned.
+ *
+ * @deprecated This method circumvents explicit configuration of the
+ * {@link KeyBuilder} and is used nearly exclusively by unit
+ * tests. While explicit configuration is not required for keys
+ * which do not include Unicode sort key components, this method
+ * also relies on a single global {@link KeyBuilder} instance
+ * protected by a lock. That lock is therefore a bottleneck. The
+ * correct practice is to use thread-local or per task
+ * {@link IKeyBuilder}s to avoid lock contention.
*/
@SuppressWarnings("unchecked")
public static final byte[] asSortKey(Object val) {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/cache/HardReferenceGlobalLRU.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/cache/HardReferenceGlobalLRU.java 2010-07-30 13:51:36 UTC (rev 3371)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/cache/HardReferenceGlobalLRU.java 2010-07-30 13:51:43 UTC (rev 3372)
@@ -57,7 +57,8 @@
* iterator. When compared to the {@link HardReferenceGlobalLRURecycler}, this
* implementation has approximately 10% higher throughput.
*
- * @version $Id$
+ * @version $Id: HardReferenceGlobalLRU.java 2799 2010-05-11 21:04:43Z
+ * thompsonbry $
* @author <a href="mailto:tho...@us...">Bryan Thompson
* </a>
* @param <K>
@@ -69,6 +70,8 @@
* {@link IDataRecordAccess} since we can not measure the bytesInMemory
* for those objects and hence the LRU eviction policy will not account
* for their memory footprint?
+ *
+ * @deprecated This implementation is not used.
*/
public class HardReferenceGlobalLRU<K, V> implements
IHardReferenceGlobalLRU<K, V> {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java 2010-07-30 13:51:36 UTC (rev 3371)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java 2010-07-30 13:51:43 UTC (rev 3372)
@@ -73,6 +73,9 @@
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
+ *
+ * @todo The key is now 100% decodable. The package should be updated to take
+ * advantage of that.
*/
public class KeyDecoder {
@@ -157,6 +160,22 @@
return a;
}
+
+ /**
+ * Return the schema name.
+ *
+ * @throws UnsupportedOperationException
+ * unless {@link SparseRowStore#schemaNameUnicodeClean} is
+ * <code>true</code>.
+ */
+ public String getSchemaName() {
+
+ if(!SparseRowStore.schemaNameUnicodeClean)
+ throw new UnsupportedOperationException();
+
+ return new String(getSchemaBytes());
+
+ }
/**
* The decoded {@link KeyType} for the primary key.
@@ -485,10 +504,17 @@
*/
public String toString() {
- return "KeyDecoder{primaryKeyType=" + primaryKeyType + ",col=" + col
- + ",timestamp=" + timestamp + ",key=" + BytesUtil.toString(key)
+ return "KeyDecoder{"
+ + (SparseRowStore.schemaNameUnicodeClean ? "schema="
+ + getSchemaName() + "," : "")//
+ + "primaryKeyType="+ primaryKeyType//
+ + (SparseRowStore.primaryKeyUnicodeClean ? ",primaryKey="
+ + getPrimaryKey() : "")//
+ + ",col=" + col //
+ + ",timestamp=" + timestamp //
+ + ",key=" + BytesUtil.toString(key) //
+ "}";
}
-
+
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java 2010-07-30 13:51:36 UTC (rev 3371)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java 2010-07-30 13:51:43 UTC (rev 3372)
@@ -128,14 +128,22 @@
if (schemaBytes == null) {
- /*
- * One time encoding of the schema name as a Unicode sort key.
- */
-
- schemaBytes = KeyBuilder.asSortKey(name);
+ if (SparseRowStore.schemaNameUnicodeClean) {
+ /*
+ * One time encoding of the schema name as UTF8.
+ */
+ try {
+ schemaBytes = name.getBytes(SparseRowStore.UTF8);
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ /*
+ * One time encoding of the schema name as a Unicode sort key.
+ */
+ schemaBytes = KeyBuilder.asSortKey(name);
+ }
-// schemaBytes = KeyBuilder.newInstance().append(name).append("\0").getKey();
-
}
return schemaBytes;
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java 2010-07-30 13:51:36 UTC (rev 3371)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java 2010-07-30 13:51:43 UTC (rev 3372)
@@ -1048,11 +1048,35 @@
*
* @author <a href="mailto:tho...@us...">Bryan
* Thompson</a>
- * @version $Id$
*/
public interface Options {
/**
+ * The schema name was originally written using a Unicode sort key.
+ * However, the JDK can generate Unicode sort keys with embedded nuls
+ * which in turn will break the logic to detect the end of the schema
+ * name in the key. In order to accommodate this behavior, the schema
+ * name is now encoded as UTF8 which also has the advantage that we can
+ * decode the schema name. Standard prefix compression on the B+Tree
+ * should make up for the larger representation of the schema name in
+ * the B+Tree.
+ * <p>
+ * This change was introduced on 7/29/2010 in the trunk. When this
+ * property is <code>true</code> it breaks compatibility with earlier
+ * revisions of the {@link SparseRowStore}. This flag may be set to
+ * <code>false</code> for backward compatibility.
+ *
+ * @see #DEFAULT_SCHEMA_NAME_UNICODE_CLEAN
+ */
+ String SCHEMA_NAME_UNICODE_CLEAN = Schema.class.getName()
+ + ".schemaName.unicodeClean";
+
+ /**
+ * @see https://sourceforge.net/apps/trac/bigdata/ticket/107
+ */
+ String DEFAULT_SCHEMA_NAME_UNICODE_CLEAN = "false";
+
+ /**
* The primary key was originally written using a Unicode sort key.
* However, the JDK generates Unicode sort keys with embedded nuls and
* that broke the logic to detect the end of the Unicode primary keys.
@@ -1083,6 +1107,17 @@
* This is a global option since it was always <code>false</code> for
* historical stores.
*
+ * @see Options#SCHEMA_NAME_UNICODE_CLEAN
+ */
+ final static transient boolean schemaNameUnicodeClean = Boolean
+ .valueOf(System.getProperty(
+ SparseRowStore.Options.SCHEMA_NAME_UNICODE_CLEAN,
+ SparseRowStore.Options.DEFAULT_SCHEMA_NAME_UNICODE_CLEAN));
+
+ /**
+ * This is a global option since it was always <code>false</code> for
+ * historical stores.
+ *
* @see Options#PRIMARY_KEY_UNICODE_CLEAN
*/
final static transient boolean primaryKeyUnicodeClean = Boolean
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/cache/TestAll.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/cache/TestAll.java 2010-07-30 13:51:36 UTC (rev 3371)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/cache/TestAll.java 2010-07-30 13:51:43 UTC (rev 3372)
@@ -76,7 +76,8 @@
suite.addTestSuite(TestStoreAndAddressLRUCache.class);
- suite.addTestSuite(TestHardReferenceGlobalLRU.class);
+ // Note: This implementation is not used.
+// suite.addTestSuite(TestHardReferenceGlobalLRU.class);
suite.addTestSuite(TestHardReferenceGlobalLRURecycler.class);
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestConcurrentJournal.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestConcurrentJournal.java 2010-07-30 13:51:36 UTC (rev 3371)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestConcurrentJournal.java 2010-07-30 13:51:43 UTC (rev 3372)
@@ -971,190 +971,193 @@
}
- /**
- * Test verifies that an {@link ITx#UNISOLATED} task failure does not cause
- * concurrent writers to abort. The test also verifies that the
- * {@link Checkpoint} record for the named index is NOT updated since none
- * of the tasks write anything on the index.
- *
- * @todo The assumptions for this test may have been invalidated by the
- * recent (4/29) changes to the group commit and task commit protocol
- * and this test might need to be reworked or rewritten.
+ /*
+ * @todo revisit this unit test. It's semantics appear to have aged.
*/
- public void test_writeService001() throws Exception {
-
- final Journal journal = new Journal(getProperties());
+// /**
+// * Test verifies that an {@link ITx#UNISOLATED} task failure does not cause
+// * concurrent writers to abort. The test also verifies that the
+// * {@link Checkpoint} record for the named index is NOT updated since none
+// * of the tasks write anything on the index.
+// *
+// * @todo The assumptions for this test may have been invalidated by the
+// * recent (4/29) changes to the group commit and task commit protocol
+// * and this test might need to be reworked or rewritten.
+// */
+// public void test_writeService001() throws Exception {
+//
+// final Journal journal = new Journal(getProperties());
+//
+// try {
+//
+// final String name = "test";
+//
+// // Note: checkpoint for the newly registered index.
+// final long checkpointAddr0;
+// {
+//
+// journal.registerIndex(name,new IndexMetadata(name,UUID.randomUUID()));
+//
+// journal.commit();
+//
+// checkpointAddr0 = journal.getIndex(name).getCheckpoint()
+// .getCheckpointAddr();
+//
+// }
+//
+// // the list of tasks to be run.
+// final List<AbstractTask<Object>> tasks = new LinkedList<AbstractTask<Object>>();
+//
+// // NOP
+// tasks.add(new AbstractTask(journal, ITx.UNISOLATED, name) {
+// protected String getTaskName() {
+// return "a";
+// }
+// protected Object doTask() throws Exception {
+// assertEquals(checkpointAddr0, ((BTree) getIndex(name))
+// .getCheckpoint().getCheckpointAddr());
+// return null;
+// }
+// });
+//
+// // throws exception.
+// tasks.add(new AbstractTask(journal, ITx.UNISOLATED, name) {
+// protected String getTaskName() {
+// return "b";
+// }
+// protected Object doTask() throws Exception {
+// assertEquals(checkpointAddr0, ((BTree) getIndex(name))
+// .getCheckpoint().getCheckpointAddr());
+// throw new ForcedAbortException();
+// }
+// });
+//
+// // NOP
+// tasks.add(new AbstractTask(journal, ITx.UNISOLATED, name) {
+// protected String getTaskName() {
+// return "c";
+// }
+// protected Object doTask() throws Exception {
+// assertEquals(checkpointAddr0, ((BTree) getIndex(name))
+// .getCheckpoint().getCheckpointAddr());
+// return null;
+// }
+// });
+//
+// // the commit counter before we submit the tasks.
+// final long commitCounter0 = journal.getRootBlockView()
+// .getCommitCounter();
+//
+// // the write service on which the tasks execute.
+// final WriteExecutorService writeService = journal
+// .getConcurrencyManager().getWriteService();
+//
+// // the group commit count before we submit the tasks.
+// final long groupCommitCount0 = writeService.getGroupCommitCount();
+//
+// // the abort count before we submit the tasks.
+// final long abortCount0 = writeService.getAbortCount();
+//
+// // the #of failed tasks before we submit the tasks.
+// final long failedTaskCount0 = writeService.getTaskFailedCount();
+//
+// // the #of successfully tasks before we submit the tasks.
+// final long successTaskCount0 = writeService.getTaskSuccessCount();
+//
+// // the #of successfully committed tasks before we submit the tasks.
+// final long committedTaskCount0 = writeService.getTaskCommittedCount();
+//
+// // submit the tasks and await their completion.
+// final List<Future<Object>> futures = journal.invokeAll( tasks );
+//
+// /*
+// * verify the #of commits on the journal is unchanged since nothing
+// * is written by any of these tasks.
+// *
+// * The expectation is that the tasks that succeed make it into the
+// * same commit group while the task that throws an exception does
+// * not cause the commit group to be aborted.
+// *
+// * Note: The tasks will make it into the same commit group iff the
+// * first task that completes is willing to wait for the others to
+// * join the commit group.
+// *
+// * Note: The tasks have a dependency on the same resource so they
+// * will be serialized (executed in a strict sequence).
+// */
+// assertEquals("commitCounter", commitCounter0, journal
+// .getRootBlockView().getCommitCounter());
+//
+// // however, a group commit SHOULD have been performed.
+// assertEquals("groupCommitCount", groupCommitCount0 + 1, writeService
+// .getGroupCommitCount());
+//
+// // NO aborts should have been performed.
+// assertEquals("aboutCount", abortCount0, writeService.getAbortCount());
+//
+// // ONE(1) tasks SHOULD have failed.
+// assertEquals("failedTaskCount", failedTaskCount0 + 1, writeService.
+// getTaskFailedCount());
+//
+// // TWO(2) tasks SHOULD have succeeded.
+// assertEquals("successTaskCount", successTaskCount0 + 2, writeService
+// .getTaskSuccessCount());
+//
+// // TWO(2) successfull tasks SHOULD have been committed.
+// assertEquals("committedTaskCount", committedTaskCount0 + 2, writeService
+// .getTaskCommittedCount());
+//
+// assertEquals( 3, futures.size());
+//
+// // tasks[0]
+// {
+//
+// Future f = futures.get(0);
+//
+// assertTrue(f.isDone());
+//
+// f.get(); // No exception expected.
+//
+// }
+//
+// // tasks[2]
+// {
+//
+// Future f = futures.get(2);
+//
+// assertTrue(f.isDone());
+//
+// f.get(); // No exception expected.
+//
+// }
+//
+// // tasks[1]
+// {
+//
+// Future f = futures.get(1);
+//
+// assertTrue(f.isDone());
+//
+// try {
+// f.get();
+// fail("Expecting exception");
+// } catch(ExecutionException ex) {
+// assertTrue(InnerCause.isInnerCause(ex, ForcedAbortException.class));
+// }
+//
+// }
+//
+// assertEquals(checkpointAddr0, journal.getIndex(name)
+// .getCheckpoint().getCheckpointAddr());
+//
+// } finally {
+//
+// journal.destroy();
+//
+// }
+//
+// }
- try {
-
- final String name = "test";
-
- // Note: checkpoint for the newly registered index.
- final long checkpointAddr0;
- {
-
- journal.registerIndex(name,new IndexMetadata(name,UUID.randomUUID()));
-
- journal.commit();
-
- checkpointAddr0 = journal.getIndex(name).getCheckpoint()
- .getCheckpointAddr();
-
- }
-
- // the list of tasks to be run.
- final List<AbstractTask<Object>> tasks = new LinkedList<AbstractTask<Object>>();
-
- // NOP
- tasks.add(new AbstractTask(journal, ITx.UNISOLATED, name) {
- protected String getTaskName() {
- return "a";
- }
- protected Object doTask() throws Exception {
- assertEquals(checkpointAddr0, ((BTree) getIndex(name))
- .getCheckpoint().getCheckpointAddr());
- return null;
- }
- });
-
- // throws exception.
- tasks.add(new AbstractTask(journal, ITx.UNISOLATED, name) {
- protected String getTaskName() {
- return "b";
- }
- protected Object doTask() throws Exception {
- assertEquals(checkpointAddr0, ((BTree) getIndex(name))
- .getCheckpoint().getCheckpointAddr());
- throw new ForcedAbortException();
- }
- });
-
- // NOP
- tasks.add(new AbstractTask(journal, ITx.UNISOLATED, name) {
- protected String getTaskName() {
- return "c";
- }
- protected Object doTask() throws Exception {
- assertEquals(checkpointAddr0, ((BTree) getIndex(name))
- .getCheckpoint().getCheckpointAddr());
- return null;
- }
- });
-
- // the commit counter before we submit the tasks.
- final long commitCounter0 = journal.getRootBlockView()
- .getCommitCounter();
-
- // the write service on which the tasks execute.
- final WriteExecutorService writeService = journal
- .getConcurrencyManager().getWriteService();
-
- // the group commit count before we submit the tasks.
- final long groupCommitCount0 = writeService.getGroupCommitCount();
-
- // the abort count before we submit the tasks.
- final long abortCount0 = writeService.getAbortCount();
-
- // the #of failed tasks before we submit the tasks.
- final long failedTaskCount0 = writeService.getTaskFailedCount();
-
- // the #of successfully tasks before we submit the tasks.
- final long successTaskCount0 = writeService.getTaskSuccessCount();
-
- // the #of successfully committed tasks before we submit the tasks.
- final long committedTaskCount0 = writeService.getTaskCommittedCount();
-
- // submit the tasks and await their completion.
- final List<Future<Object>> futures = journal.invokeAll( tasks );
-
- /*
- * verify the #of commits on the journal is unchanged since nothing
- * is written by any of these tasks.
- *
- * The expectation is that the tasks that succeed make it into the
- * same commit group while the task that throws an exception does
- * not cause the commit group to be aborted.
- *
- * Note: The tasks will make it into the same commit group iff the
- * first task that completes is willing to wait for the others to
- * join the commit group.
- *
- * Note: The tasks have a dependency on the same resource so they
- * will be serialized (executed in a strict sequence).
- */
- assertEquals("commitCounter", commitCounter0, journal
- .getRootBlockView().getCommitCounter());
-
- // however, a group commit SHOULD have been performed.
- assertEquals("groupCommitCount", groupCommitCount0 + 1, writeService
- .getGroupCommitCount());
-
- // NO aborts should have been performed.
- assertEquals("aboutCount", abortCount0, writeService.getAbortCount());
-
- // ONE(1) tasks SHOULD have failed.
- assertEquals("failedTaskCount", failedTaskCount0 + 1, writeService.
- getTaskFailedCount());
-
- // TWO(2) tasks SHOULD have succeeded.
- assertEquals("successTaskCount", successTaskCount0 + 2, writeService
- .getTaskSuccessCount());
-
- // TWO(2) successfull tasks SHOULD have been committed.
- assertEquals("committedTaskCount", committedTaskCount0 + 2, writeService
- .getTaskCommittedCount());
-
- assertEquals( 3, futures.size());
-
- // tasks[0]
- {
-
- Future f = futures.get(0);
-
- assertTrue(f.isDone());
-
- f.get(); // No exception expected.
-
- }
-
- // tasks[2]
- {
-
- Future f = futures.get(2);
-
- assertTrue(f.isDone());
-
- f.get(); // No exception expected.
-
- }
-
- // tasks[1]
- {
-
- Future f = futures.get(1);
-
- assertTrue(f.isDone());
-
- try {
- f.get();
- fail("Expecting exception");
- } catch(ExecutionException ex) {
- assertTrue(InnerCause.isInnerCause(ex, ForcedAbortException.class));
- }
-
- }
-
- assertEquals(checkpointAddr0, journal.getIndex(name)
- .getCheckpoint().getCheckpointAddr());
-
- } finally {
-
- journal.destroy();
-
- }
-
- }
-
/**
* Test verifies that a write on an index will cause the index to be
* checkpointed when the task completes.
@@ -1206,262 +1209,265 @@
}
}
-
- /**
- * Test verifies that a task failure causes accessed indices to be rolled
- * back to their last checkpoint.
- *
- * FIXME write test where a task registers an index and then throws an
- * exception. This will cause the index to have a checkpoint record that
- * does not agree with {@link Name2Addr} for the last commit point. Verify
- * that the index is not in fact available to another task that is executed
- * after the failed task (it will be if we merely close the index and then
- * re-open it since it will reopen from the last checkpoint NOT from the
- * last commit point).
- *
- * FIXME write test where a tasks (a), (b) and (c) are submitted with
- * invokeAll() in that order and require a lock on the same index. Task (a)
- * writes on an existing index and completes normally. The index SHOULD be
- * checkpointed and task (b) SHOULD be able to read the data written in task
- * (a) and SHOULD be run in the same commit group. Task (b) then throws an
- * exception. Verify that the index is rolledback to the checkpoint for (a)
- * (vs the last commit point) using task (c) which will read on the same
- * index looking for the correct checkpoint record and data in the index.
- * This test will fail if (b) is not reading from the checkpoint written by
- * (a) or if (c) reads from the last commit point rather than the checkpoint
- * written by (a).
- *
- * FIXME write tests to verify that an {@link #abort()} causes all running
- * tasks to be interrupted and have their write sets discarded (should it?
- * Should an abort just be an shutdownNow() in response to some truely nasty
- * problem?)
+
+ /*
+ * @todo revisit this unit test. It's semantics appear to have aged.
*/
- public void test_writeService002()throws Exception {
-
- final Properties properties = new Properties(getProperties());
-
- /*
- * Note: restricting the thread pool size does not give us the control
- * that we need because it results in each task running as its own
- * commit group.
- */
+// /**
+// * Test verifies that a task failure causes accessed indices to be rolled
+// * back to their last checkpoint.
+// *
+// * FIXME write test where a task registers an index and then throws an
+// * exception. This will cause the index to have a checkpoint record that
+// * does not agree with {@link Name2Addr} for the last commit point. Verify
+// * that the index is not in fact available to another task that is executed
+// * after the failed task (it will be if we merely close the index and then
+// * re-open it since it will reopen from the last checkpoint NOT from the
+// * last commit point).
+// *
+// * FIXME write test where a tasks (a), (b) and (c) are submitted with
+// * invokeAll() in that order and require a lock on the same index. Task (a)
+// * writes on an existing index and completes normally. The index SHOULD be
+// * checkpointed and task (b) SHOULD be able to read the data written in task
+// * (a) and SHOULD be run in the same commit group. Task (b) then throws an
+// * exception. Verify that the index is rolledback to the checkpoint for (a)
+// * (vs the last commit point) using task (c) which will read on the same
+// * index looking for the correct checkpoint record and data in the index.
+// * This test will fail if (b) is not reading from the checkpoint written by
+// * (a) or if (c) reads from the last commit point rather than the checkpoint
+// * written by (a).
+// *
+// * FIXME write tests to verify that an {@link #abort()} causes all running
+// * tasks to be interrupted and have their write sets discarded (should it?
+// * Should an abort just be an shutdownNow() in response to some truely nasty
+// * problem?)
+// */
+// public void test_writeService002()throws Exception {
+//
+// final Properties properties = new Properties(getProperties());
+//
// /*
-// * Note: Force the write service to be single threaded so that we can
-// * control the order in which the tasks start by the order in which they
-// * are submitted.
+// * Note: restricting the thread pool size does not give us the control
+// * that we need because it results in each task running as its own
+// * commit group.
// */
-// properties.setProperty(Options.WRITE_SERVICE_CORE_POOL_SIZE,"1");
-// properties.setProperty(Options.WRITE_SERVICE_MAXIMUM_POOL_SIZE,"1");
-
- final Journal journal = new Journal(properties);
-
- try {
-
- final String name = "test";
-
- // Note: checkpoint for the newly registered index.
- final long checkpointAddr0;
- {
-
- // register
- journal.registerIndex(name);
-
- // commit.
- journal.commit();
-
- // note checkpoint for index.
- checkpointAddr0 = journal.getIndex(name).getCheckpoint()
- .getCheckpointAddr();
-
- }
-
- // Note: commit counter before we invoke the tasks.
- final long commitCounter = journal.getRootBlockView()
- .getCommitCounter();
-
- final WriteExecutorService writeService = journal
- .getConcurrencyManager().getWriteService();
-
- // Note: group commit counter before we invoke the tasks.
- final long groupCommitCount0 = writeService.getGroupCommitCount();
-
- // Note: #of failed tasks before we submit the tasks.
- final long failedTaskCount0 = writeService.getTaskFailedCount();
- final long successTaskCount0 = writeService.getTaskSuccessCount();
- final long committedTaskCount0 = writeService.getTaskCommittedCount();
-
- // Note: set by one of the tasks below.
- final AtomicLong checkpointAddr2 = new AtomicLong(0L);
-
- final AtomicReference<Future<? extends Object>> futureB = new AtomicReference<Future<? extends Object>>();
- final AtomicReference<Future<? extends Object>> futureC = new AtomicReference<Future<? extends Object>>();
- final AtomicReference<Future<? extends Object>> futureD = new AtomicReference<Future<? extends Object>>();
-
- /*
- * Note: the setup for this test is a PITA. In order to exert full
- * control over the order in which the tasks begin to execute we
- * need to have each task submit the next itself. This is because it
- * is possible for any of these tasks to be the first one to grab
- * the exclusive lock on the necessary resource [name]. We can't
- * solve this problem by restricting the #of threads that can run
- * the tasks since that limits the size of the commit group. So we
- * are stuck imposing serial execution using the behavior of the
- * tasks themselves.
- *
- * Create the task objects in the reverse order of their execution.
- */
-
- // task (d) verifies expected rollback checkpoint was restored.
- final AbstractTask d = new AbstractTask(journal,ITx.UNISOLATED,name){
- protected String getTaskName() {return "d";}
- protected Object doTask() throws Exception {
- // commit counter unchanged.
- assertEquals("commitCounter", commitCounter, getJournal()
- .getRootBlockView().getCommitCounter());
- if(checkpointAddr2.get()==0L) {
- fail("checkpointAddr2 was not set");
- }
- // lookup index.
- BTree ndx = (BTree)getIndex(name);
- final long newCheckpointAddr =ndx.getCheckpoint().getCheckpointAddr();
- // verify checkpoint != last committed checkpoint.
- assertNotSame(checkpointAddr0,newCheckpointAddr);
- // verify checkpoint == last rollback checkpoint.
- assertEquals(checkpointAddr2.get(),newCheckpointAddr);
- return null;
- }
- };
-
- /*
- * task (c) notes the last checkpoint, writes on the index, and then
- * fails. This is designed to trigger rollback of the index to the
- * last checkpoint, which is the checkpoint that we note at the
- * start of this task.
- */
- final AbstractTask c = new AbstractTask(journal,ITx.UNISOLATED,name){
- protected String getTaskName() {return "c";}
- protected Object doTask() throws Exception {
- // commit counter unchanged.
- assertEquals("commitCounter", commitCounter, getJournal()
- .getRootBlockView().getCommitCounter());
- // lookup index.
- BTree ndx = (BTree)getIndex(name);
- // note the last checkpoint written.
- final long newCheckpointAddr = ndx.getCheckpoint().getCheckpointAddr();
- assertNotSame(0L,newCheckpointAddr);
- assertNotSame(checkpointAddr0,newCheckpointAddr);
- // make note of the checkpoint before we force an abort.
- assertTrue("checkpointAddr2 already set?",checkpointAddr2.compareAndSet(0L, newCheckpointAddr));
- // write another record on the index.
- ndx.insert(new byte[]{3}, new byte[]{3});
- // run task (d) next.
- assertTrue(futureD.compareAndSet(null,journal.submit(d)));
- // force task to about with dirty index.
- throw new ForcedAbortException();
- }
- };
-
- // task (b) writes another record on the index.
- final AbstractTask b = new AbstractTask(journal,ITx.UNISOLATED,name){
- protected String getTaskName() {return "b";}
- protected Object doTask() throws Exception {
- // commit counter unchanged.
- assertEquals("commitCounter", commitCounter, getJournal()
- .getRootBlockView().getCommitCounter());
- // lookup index.
- BTree ndx = (BTree)getIndex(name);
- // verify checkpoint was updated.
- assertNotSame(checkpointAddr0,ndx.getCheckpoint().getCheckpointAddr());
- // write another record on the index.
- ndx.insert(new byte[]{2}, new byte[]{2});
- // run task (c) next.
- assertTrue(futureC.compareAndSet(null,journal.submit(c)));
- return null;
- }
- };
-
- // task (a) writes on index.
- final AbstractTask a = new AbstractTask(journal,ITx.UNISOLATED,name){
- protected String getTaskName() {return "a";}
- protected Object doTask() throws Exception {
- // commit counter unchanged.
- assertEquals("commitCounter", commitCounter, getJournal()
- .getRootBlockView().getCommitCounter());
- // group commit counter unchanged.
- assertEquals("groupCommitCounter", groupCommitCount0,
- writeService.getGroupCommitCount());
- // lookup index.
- BTree ndx = (BTree)getIndex(name);
- // verify same checkpoint.
- assertEquals(checkpointAddr0,ndx.getCheckpoint().getCheckpointAddr());
- // write record on the index.
- ndx.insert(new byte[]{1}, new byte[]{1});
- // run task (b) next.
- assertTrue(futureB.compareAndSet(null,journal.submit(b)));
- return null;
- }
- };
-
-// final List<AbstractTask> tasks = Arrays.asList(new AbstractTask[] {
-// a,b,c,d
-// });
+//// /*
+//// * Note: Force the write service to be single threaded so that we can
+//// * control the order in which the tasks start by the order in which they
+//// * are submitted.
+//// */
+//// properties.setProperty(Options.WRITE_SERVICE_CORE_POOL_SIZE,"1");
+//// properties.setProperty(Options.WRITE_SERVICE_MAXIMUM_POOL_SIZE,"1");
+//
+// final Journal journal = new Journal(properties);
+//
+// try {
+//
+// final String name = "test";
+//
+// // Note: checkpoint for the newly registered index.
+// final long checkpointAddr0;
+// {
+//
+// // register
+// journal.registerIndex(name);
+//
+// // commit.
+// journal.commit();
+//
+// // note checkpoint for index.
+// checkpointAddr0 = journal.getIndex(name).getCheckpoint()
+// .getCheckpointAddr();
+//
+// }
+//
+// // Note: commit counter before we invoke the tasks.
+// final long commitCounter = journal.getRootBlockView()
+// .getCommitCounter();
+//
+// final WriteExecutorService writeService = journal
+// .getConcurrencyManager().getWriteService();
+//
+// // Note: group commit counter before we invoke the tasks.
+// final long groupCommitCount0 = writeService.getGroupCommitCount();
+//
+// // Note: #of failed tasks before we submit the tasks.
+// final long failedTaskCount0 = writeService.getTaskFailedCount();
+// final long successTaskCount0 = writeService.getTaskSuccessCount();
+// final long committedTaskCount0 = writeService.getTaskCommittedCount();
+//
+// // Note: set by one of the tasks below.
+// final AtomicLong checkpointAddr2 = new AtomicLong(0L);
+//
+// final AtomicReference<Future<? extends Object>> futureB = new AtomicReference<Future<? extends Object>>();
+// final AtomicReference<Future<? extends Object>> futureC = new AtomicReference<Future<? extends Object>>();
+// final AtomicReference<Future<? extends Object>> futureD = new AtomicReference<Future<? extends Object>>();
//
-// final List<Future<Object>> futures = journal.invokeAll( tasks );
-
- final Future<? extends Object> futureA = journal.submit( a );
-
- /*
- * wait for (a). if all tasks are in the same commit group then all
- * tasks will be done once we have the future for (a).
- */
- futureA.get(); // task (a)
-
- /*
- * The expectation is that the tasks that succeed make it into the
- * same commit group while the task that throws an exception does
- * not cause the commit group to be aborted. Therefore there should
- * be ONE (1) commit more than when we submitted the tasks.
- *
- * Note: The tasks will make it into the same commit group iff the
- * first task that completes is willing to wait for the others to
- * join the commit group.
- *
- * Note: The tasks have a dependency on the same resource so they
- * will be serialized (executed in a strict sequence).
- */
- assertEquals("failedTaskCount", failedTaskCount0 + 1,
- writeService.getTaskFailedCount());
- assertEquals("successTaskCount", successTaskCount0 + 3,
- writeService.getTaskSuccessCount());
- assertEquals("committedTaskCount", committedTaskCount0 + 3,
- writeService.getTaskCommittedCount());
- assertEquals("groupCommitCount", groupCommitCount0 + 1,
- writeService.getGroupCommitCount());
- assertEquals("commitCounter", commitCounter + 1, journal
- .getRootBlockView().getCommitCounter());
-
-// assertEquals( 4, futures.size());
-
- futureB.get().get(); // task (b)
- {
- // task (c) did the abort.
- Future f = futureC.get();
- try {f.get(); fail("Expecting exception");}
- catch(ExecutionException ex) {
- if(!InnerCause.isInnerCause(ex, ForcedAbortException.class)) {
- fail("Expecting "+ForcedAbortException.class+", not "+ex, ex);
- }
- }
- }
- futureD.get().get(); // task (d)
-
- } finally {
-
- journal.destroy();
-
- }
-
- }
+// /*
+// * Note: the setup for this test is a PITA. In order to exert full
+// * control over the order in which the tasks begin to execute we
+// * need to have each task submit the next itself. This is because it
+// * is possible for any of these tasks to be the first one to grab
+// * the exclusive lock on the necessary resource [name]. We can't
+// * solve this problem by restricting the #of threads that can run
+// * the tasks since that limits the size of the commit group. So we
+// * are stuck imposing serial execution using the behavior of the
+// * tasks themselves.
+// *
+// * Create the task objects in the reverse order of their execution.
+// */
+//
+// // task (d) verifies expected rollback checkpoint was restored.
+// final AbstractTask d = new AbstractTask(journal,ITx.UNISOLATED,name){
+// protected String getTaskName() {return "d";}
+// protected Object doTask() throws Exception {
+// // commit counter unchanged.
+// assertEquals("commitCounter", commitCounter, getJournal()
+// .getRootBlockView().getCommitCounter());
+// if(checkpointAddr2.get()==0L) {
+// fail("checkpointAddr2 was not set");
+// }
+// // lookup index.
+// BTree ndx = (BTree)getIndex(name);
+// final long newCheckpointAddr =ndx.getCheckpoint().getCheckpointAddr();
+// // verify checkpoint != last committed checkpoint.
+// assertNotSame(checkpointAddr0,newCheckpointAddr);
+// // verify checkpoint == last rollback checkpoint.
+// assertEquals(checkpointAddr2.get(),newCheckpointAddr);
+// return null;
+// }
+// };
+//
+// /*
+// * task (c) notes the last checkpoint, writes on the index, and then
+// * fails. This is designed to trigger rollback of the index to the
+// * last checkpoint, which is the checkpoint that we note at the
+// * start of this task.
+// */
+// final AbstractTask c = new AbstractTask(journal,ITx.UNISOLATED,name){
+// protected String getTaskName() {return "c";}
+// protected Object doTask() throws Exception {
+// // commit counter unchanged.
+// assertEquals("commitCounter", commitCounter, getJournal()
+// .getRootBlockView().getCommitCounter());
+// // lookup index.
+// BTree ndx = (BTree)getIndex(name);
+// // note the last checkpoint written.
+// final long newCheckpointAddr = ndx.getCheckpoint().getCheckpointAddr();
+// assertNotSame(0L,newCheckpointAddr);
+// assertNotSame(checkpointAddr0,newCheckpointAddr);
+// // make note of the checkpoint before we force an abort.
+// assertTrue("checkpointAddr2 already set?",checkpointAddr2.compareAndSet(0L, newCheckpointAddr));
+// // write another record on the index.
+// ndx.insert(new byte[]{3}, new byte[]{3});
+// // run task (d) next.
+// assertTrue(futureD.compareAndSet(null,journal.submit(d)));
+// // force task to about with dirty index.
+// throw new ForcedAbortException();
+// }
+// };
+//
+// // task (b) writes another record on the index.
+// final AbstractTask b = new AbstractTask(journal,ITx.UNISOLATED,name){
+// protected String getTaskName() {return "b";}
+// protected Object doTask() throws Exception {
+// // commit counter unchanged.
+// assertEquals("commitCounter", commitCounter, getJournal()
+// .getRootBlockView().getCommitCounter());
+// // lookup index.
+// BTree ndx = (BTree)getIndex(name);
+// // verify checkpoint was updated.
+// assertNotSame(checkpointAddr0,ndx.getCheckpoint().getCheckpointAddr());
+// // write another record on the index.
+// ndx.insert(new byte[]{2}, new byte[]{2});
+// // run task (c) next.
+// assertTrue(futureC.compareAndSet(null,journal.submit(c)));
+// return null;
+// }
+// };
+//
+// // task (a) writes on index.
+// final AbstractTask a = new AbstractTask(journal,ITx.UNISOLATED,name){
+// protected String getTaskName() {return "a";}
+// protected Object doTask() throws Exception {
+// // commit counter unchanged.
+// assertEquals("commitCounter", commitCounter, getJournal()
+// .getRootBlockView().getCommitCounter());
+// // group commit counter unchanged.
+// assertEquals("groupCommitCounter", groupCommitCount0,
+// writeService.getGroupCommitCount());
+// // lookup index.
+// BTree ndx = (BTree)getIndex(name);
+// // verify same checkpoint.
+// assertEquals(checkpointAddr0,ndx.getCheckpoint().getCheckpointAddr());
+// // write record on the index.
+// ndx.insert(new byte[]{1}, new byte[]{1});
+// // run task (b) next.
+// assertTrue(futureB.compareAndSet(null,journal.submit(b)));
+// return null;
+// }
+// };
+//
+//// final List<AbstractTask> tasks = Arrays.asList(new AbstractTask[] {
+//// a,b,c,d
+//// });
+////
+//// final List<Future<Object>> futures = journal.invokeAll( tasks );
+//
+// final Future<? extends Object> futureA = journal.submit( a );
+//
+// /*
+// * wait for (a). if all tasks are in the same commit group then all
+// * tasks will be done once we have the future for (a).
+// */
+// futureA.get(); // task (a)
+//
+// /*
+// * The expectation is that the tasks that succeed make it into the
+// * same commit group while the task that throws an exception does
+// * not cause the commit group to be aborted. Therefore there should
+// * be ONE (1) commit more than when we submitted the tasks.
+// *
+// * Note: The tasks will make it into the same commit group iff the
+// * first task that completes is willing to wait for the others to
+// * join the commit group.
+// *
+// * Note: The tasks have a dependency on the same resource so they
+// * will be serialized (executed in a strict sequence).
+// */
+// assertEquals("failedTaskCount", failedTaskCount0 + 1,
+// writeService.getTaskFailedCount());
+// assertEquals("successTaskCount", successTaskCount0 + 3,
+// writeService.getTaskSuccessCount());
+// assertEquals("committedTaskCount", committedTaskCount0 + 3,
+// writeService.getTaskCommittedCount());
+// assertEquals("groupCommitCount", groupCommitCount0 + 1,
+// writeService.getGroupCommitCount());
+// assertEquals("commitCounter", commitCounter + 1, journal
+// .getRootBlockView().getCommitCounter());
+//
+//// assertEquals( 4, futures.size());
+//
+// futureB.get().get(); // task (b)
+// {
+// // task (c) did the abort.
+// Future f = futureC.get();
+// try {f.get(); fail("Expecting exception");}
+// catch(ExecutionException ex) {
+// if(!InnerCause.isInnerCause(ex, ForcedAbortException.class)) {
+// fail("Expecting "+ForcedAbortException.class+", not "+ex, ex);
+// }
+// }
+// }
+// futureD.get().get(); // task (d)
+//
+// } finally {
+//
+// journal.destroy();
+//
+// }
+//
+// }
/**
* A class used to force aborts on tasks and then recognize the abort by the
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestJournalBasics.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestJournalBasics.java 2010-07-30 13:51:36 UTC (rev 3371)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestJournalBasics.java 2010-07-30 13:51:43 UTC (rev 3372)
@@ -104,8 +104,9 @@
// test basics of the concurrent task execution.
suite.addTestSuite(TestConcurrentJournal.class);
- // test tasks to add and drop named indices.
- suite.addTestSuite(TestAddDropIndexTask.class);
+// test tasks to add and drop named indices.
+// This has been commented out since the unit test has dated semantics.
+// suite.addTestSuite(TestAddDropIndexTask.class);
// test writing on one or more unisolated indices and verify read back after the commit.
suite.addTestSuite(TestUnisolatedWriteTasks.class);
// stress test of throughput when lock contention serializes unisolated writers.
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/relation/locator/TestDefaultResourceLocator.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/relation/locator/TestDefaultResourceLocator.java 2010-07-30 13:51:36 UTC (rev 3371)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/relation/locator/TestDefaultResourceLocator.java 2010-07-30 13:51:43 UTC (rev 3372)
@@ -163,9 +163,12 @@
namespace, ITx.UNISOLATED)) == mockRelation);
/*
- * the read-committed view still does not see the relation since
- * there has not been a commit yet after the index was created.
+ * @todo The read-committed view still does not see the relation
+ * since there has not been a commit yet after the index was
+ * created.
*/
+ if(false) {
+
assertNull(((MockRelation) store.getResourceLocator().locate(
namespace, ITx.READ_COMMITTED)));
@@ -207,6 +210,8 @@
assertTrue(readCommittedView2 == (MockRelation) store
.getResourceLocator().locate(namespace,
ITx.READ_COMMITTED));
+
+ }
}
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/search/TestKeyBuilder.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/search/TestKeyBuilder.java 2010-07-30 13:51:36 UTC (rev 3371)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/search/TestKeyBuilder.java 2010-07-30 13:51:43 UTC (rev 3372)
@@ -183,74 +183,77 @@
+ BytesUtil.toString(b));
}
-
- /**
- * @todo this test needs to populate an index with terms that would match if
- * we were allowing a prefix match and then verify that the terms are
- * NOT matched. it should also verify that terms that are exact
- * matches are matched.
- *
- * @todo also test ability to extract the docId and fieldId from the key.
- *
- * @todo refactor into an {@link ITupleSerializer}.
- *
- * @todo make the fieldId optional in the key. this needs to be part o...
[truncated message content] |
|
From: <btm...@us...> - 2010-07-30 20:41:41
|
Revision: 3377
http://bigdata.svn.sourceforge.net/bigdata/?rev=3377&view=rev
Author: btmurphy
Date: 2010-07-30 20:41:35 +0000 (Fri, 30 Jul 2010)
Log Message:
-----------
[branch dev-btm]: fixed a typo in JiniServicesHelper that prevented TestBigdataClient from passing when run on eclipse
Modified Paths:
--------------
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/JiniServicesHelper.java
branches/dev-btm/bigdata-jini/src/resources/config/bigdataStandaloneTesting.config
branches/dev-btm/src/resources/config/bigdataCluster.config
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/JiniServicesHelper.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/JiniServicesHelper.java 2010-07-30 18:49:37 UTC (rev 3376)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/JiniServicesHelper.java 2010-07-30 20:41:35 UTC (rev 3377)
@@ -586,7 +586,7 @@
optionsList.add(options[i]);
}
String joinGroupsOverrideStr =
- "com.bigdata.loadblancer.groupsToJoin=new String[] "
+ "com.bigdata.loadbalancer.groupsToJoin=new String[] "
+"{"
+"\""+fedname+"\""
+"}";
Modified: branches/dev-btm/bigdata-jini/src/resources/config/bigdataStandaloneTesting.config
===================================================================
--- branches/dev-btm/bigdata-jini/src/resources/config/bigdataStandaloneTesting.config 2010-07-30 18:49:37 UTC (rev 3376)
+++ branches/dev-btm/bigdata-jini/src/resources/config/bigdataStandaloneTesting.config 2010-07-30 20:41:35 UTC (rev 3377)
@@ -36,6 +36,14 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeUnit.*;
+import java.net.NetworkInterface;
+import net.jini.constraint.BasicMethodConstraints;
+import net.jini.core.constraint.ConnectionRelativeTime;
+import net.jini.core.constraint.InvocationConstraints;
+import net.jini.discovery.LookupDiscoveryManager;
+import com.bigdata.util.config.NicUtil;
+import com.bigdata.util.config.ConfigDeployUtil;
+
/*
* This is a standalone configuration file for a bigdata federation
* used for unit tests in conjunction with the JiniServicesHelper class.
@@ -431,6 +439,61 @@
}
+com.bigdata.loadbalancer {
+
+ private static exportIpAddr =
+ NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
+ private static exportPort =
+ Integer.parseInt( System.getProperty("exportPort", "0") );
+
+ groupsToJoin = ConfigDeployUtil.getGroupsToDiscover();
+ locatorsToJoin = ConfigDeployUtil.getLocatorsToDiscover();
+
+ private static exporterTcpServerEndpoint =
+ TcpServerEndpoint.getInstance(exportIpAddr, exportPort);
+ private static serverILFactory =
+ new BasicILFactory(
+ new BasicMethodConstraints(
+ new InvocationConstraints(
+ new ConnectionRelativeTime(
+ ConfigDeployUtil.getLong(
+ "rmi.connectTimeout")),
+ null)),
+ null);
+
+ serverExporter =
+ new BasicJeriExporter
+ (TcpServerEndpoint.getInstance(exportIpAddr,exportPort),
+ serverILFactory,
+ false,
+ true);
+
+ discoveryManager = new LookupDiscoveryManager(groupsToJoin,
+ locatorsToJoin,
+ null,
+ this);
+ // Where service state is persisted
+ persistenceDirectory =
+ ConfigUtil.concat
+ ( new String[] { System.getProperty("app.home", "${user.dir}"),
+ "${/}var${/}state${/}loadbalancerState" } );
+}
+
+net.jini.discovery.LookupDiscovery {
+ multicastRequestHost = com.bigdata.loadbalancer.exportIpAddr;
+ multicastInterfaces = new NetworkInterface[] {
+ NicUtil.getNetworkInterface(com.bigdata.loadbalancer.exportIpAddr)
+ };
+}
+
+net.jini.lookup.ServiceDiscoveryManager {
+ eventListenerExporter =
+ new BasicJeriExporter
+ (com.bigdata.loadbalancer.exporterTcpServerEndpoint,
+ com.bigdata.loadbalancer.serverILFactory, false, false);
+}
+
+
/*
* Note: Whitespace intentionally follows this comment!
*/
Modified: branches/dev-btm/src/resources/config/bigdataCluster.config
===================================================================
--- branches/dev-btm/src/resources/config/bigdataCluster.config 2010-07-30 18:49:37 UTC (rev 3376)
+++ branches/dev-btm/src/resources/config/bigdataCluster.config 2010-07-30 20:41:35 UTC (rev 3377)
@@ -1201,7 +1201,6 @@
"-Dlog4j.primary.configuration=@NAS@/dist/bigdata/var/config/logging/loadbalancer-logging.properties",
"-DusingServicesManagerService=true",
"-Dbigdata.logDir=@NAS@/dist/bigdata/var/log",
- "-DappHome=@NAS@/dist/bigdata",
"-DappHome=@APP_HOME@",
"-Dconfig=@NAS@/dist/bigdata/var/config/jini/loadbalancer.config"
};
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <btm...@us...> - 2010-07-30 22:46:27
|
Revision: 3380
http://bigdata.svn.sourceforge.net/bigdata/?rev=3380&view=rev
Author: btmurphy
Date: 2010-07-30 22:46:19 +0000 (Fri, 30 Jul 2010)
Log Message:
-----------
merge -r:3370:HEAD(3378) ~/bigdata/trunk ~/bigdata/branches/dev-btm [trunk --> branch dev-btm]
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/btree/BigdataMap.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/IKeyBuilder.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractBTreeTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractTupleCursorTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestBTreeLeafCursors.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestBigdataMap.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestChunkedIterators.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestCopyOnWrite.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestDirtyIterators.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIncrementalWrite.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexSegmentBuilderCacheInteraction.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexSegmentBuilderWithCompactingMerge.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexSegmentBuilderWithIncrementalBuild.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexSegmentBuilderWithSmallTree.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexSegmentWithBloomFilter.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestInsertLookupRemoveKeysInRootLeaf.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIterators.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestLinearListMethods.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestMutableBTreeCursors.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestReopen.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestSplitJoinRootLeaf.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestSplitJoinThreeLevels.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestSplitRootLeaf.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestTouch.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestTransientBTree.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/filter/TestTupleFilters.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/AbstractUnicodeKeyBuilderTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestSuccessorUtil.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/raba/codec/AbstractRabaCoderTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/raba/codec/RandomURIGenerator.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestBuildTask.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestBuildTask2.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestMergeTask.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestOverflow.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestResourceManagerBootstrap.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestSegSplitter.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestMove.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestRangeQuery.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestRestartSafe.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestScatterSplit.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestSplitJoin.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/ndx/pipeline/TestMasterTaskWithSplits.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/TestBigdataClient.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/internal/BlobOverflowHandler.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPO.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPORelation.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOTupleSerializer.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOValueCoders.java
Property Changed:
----------------
branches/dev-btm/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/attr/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/disco/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/
branches/dev-btm/bigdata-perf/
branches/dev-btm/bigdata-perf/lubm/lib/
branches/dev-btm/bigdata-perf/lubm/src/resources/
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util/
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/
branches/dev-btm/dsi-utils/LEGAL/
branches/dev-btm/dsi-utils/lib/
branches/dev-btm/dsi-utils/src/
branches/dev-btm/dsi-utils/src/test/
branches/dev-btm/dsi-utils/src/test/it/
branches/dev-btm/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/osgi/
Property changes on: branches/dev-btm
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/LEXICON_REFACTOR_BRANCH:2633-3304
/branches/bugfix-btm:2594-3237
/branches/fko:3150-3194
/trunk:2575-2594,2596-2877,2882-2903,2910-3370
+ /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/LEXICON_REFACTOR_BRANCH:2633-3304
/branches/bugfix-btm:2594-3237
/branches/fko:3150-3194
/trunk:2575-2594,2596-2877,2882-2903,2910-3378
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/BigdataMap.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/BigdataMap.java 2010-07-30 22:14:17 UTC (rev 3379)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/BigdataMap.java 2010-07-30 22:46:19 UTC (rev 3380)
@@ -58,21 +58,21 @@
* Note: The total order of the {@link BigdataMap} is completely determined by
* {@link ITupleSerializer#serializeKey(Object)}. There is NO concept of a
* {@link Comparator}. The {@link ITupleSerializer} is responsible for coercing
- * application keys into variable length <strong>unsigned</strong> byte[]s
- * which are the keys for the underlying B+Tree. The order for the B+Tree is the
+ * application keys into variable length <strong>unsigned</strong> byte[]s which
+ * are the keys for the underlying B+Tree. The order for the B+Tree is the
* natural order for the <strong>unsigned byte[]</strong>s. {@link KeyBuilder}
* supports the generation of unsigned byte[]s from various kinds of Java
* primitives and Unicode {@link String}s and is typically used to write the
* {@link ITupleSerializer#serializeKey(Object)} method.
* <p>
- * Note: The coercion of the application keys into unsigned byte[]s is not
- * typesafe unless you either consistently use a strongly typed instance of this
+ * Note: The coercion of the application keys into unsigned byte[]s is not type
+ * safe unless you either consistently use a strongly typed instance of this
* class or specify an {@link ITupleSerializer} for the backing B+Tree that only
* allows application keys that are instances of acceptable classes. This issue
* is more critical for keys than for values since the keys define the total
* index order and the default coercion rules for keys are provided by
- * {@link KeyBuilder#asSortKey(Object)} which does not attenpt to partition the
- * key space by the application key type (keys are not safely polymorphic by
+ * {@link KeyBuilder#append(Object)} which does not attempt to partition the key
+ * space by the application key type (keys are not safely polymorphic by
* default).
* <p>
* Note: When storing Java objects in the tuple value, the value MUST be treated
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/IKeyBuilder.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/IKeyBuilder.java 2010-07-30 22:14:17 UTC (rev 3379)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/IKeyBuilder.java 2010-07-30 22:46:19 UTC (rev 3380)
@@ -33,6 +33,7 @@
import java.util.Locale;
import java.util.Properties;
import java.util.UUID;
+
import com.bigdata.btree.BytesUtil;
import com.bigdata.btree.keys.KeyBuilder.Options;
@@ -88,7 +89,6 @@
* {@link #appendText(String, boolean, boolean)}.
* </p>
*
- * @see KeyBuilder#asSortKey(Object)
* @see KeyBuilder#newInstance()
* @see KeyBuilder#newUnicodeInstance()
* @see KeyBuilder#newUnicodeInstance(Properties)
@@ -97,7 +97,7 @@
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
*/
-public interface IKeyBuilder extends ISortKeyBuilder {
+public interface IKeyBuilder extends ISortKeyBuilder<Object> {
/**
* The #of bytes of data in the key.
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-07-30 22:14:17 UTC (rev 3379)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-07-30 22:46:19 UTC (rev 3380)
@@ -1065,78 +1065,8 @@
}
- /*
- * static helper methods.
- */
-
- /**
- * Used to unbox an application key (convert it to an unsigned byte[]).
- */
- static private final IKeyBuilder _keyBuilder = newUnicodeInstance();
-
- /**
- * Utility method converts an application key to a sort key (an unsigned
- * byte[] that imposes the same sort order).
- * <p>
- * Note: This method is thread-safe.
- * <p>
- * Note: Strings are Unicode safe for the default locale. See
- * {@link Locale#getDefault()}. If you require a specific local or different
- * locals at different times or for different indices then you MUST
- * provision and apply your own {@link KeyBuilder}.
- *
- * @param val
- * An application key.
- *
- * @return The unsigned byte[] equivalent of that key. This will be
- * <code>null</code> iff the <i>key</i> is <code>null</code>. If the
- * <i>key</i> is a byte[], then the byte[] itself will be returned.
- *
- * @deprecated This method circumvents explicit configuration of the
- * {@link KeyBuilder} and is used nearly exclusively by unit
- * tests. While explicit configuration is not required for keys
- * which do not include Unicode sort key components, this method
- * also relies on a single global {@link KeyBuilder} instance
- * protected by a lock. That lock is therefore a bottleneck. The
- * correct practice is to use thread-local or per task
- * {@link IKeyBuilder}s to avoid lock contention.
- */
- @SuppressWarnings("unchecked")
- public static final byte[] asSortKey(Object val) {
+ public byte[] getSortKey(final Object val) {
- if (val == null) {
-
- return null;
-
- }
-
- if (val instanceof byte[]) {
-
- return (byte[]) val;
-
- }
-
- /*
- * Synchronize on the keyBuilder to avoid concurrent modification of its
- * state.
- */
-
- synchronized (_keyBuilder) {
-
- return _keyBuilder.getSortKey(val);
-
-// _keyBuilder.reset();
-//
-// _keyBuilder.append( key );
-//
-// return _keyBuilder.getKey();
-
- }
-
- }
-
- public byte[] getSortKey(Object val) {
-
reset();
append( val );
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java 2010-07-30 22:14:17 UTC (rev 3379)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java 2010-07-30 22:46:19 UTC (rev 3380)
@@ -141,7 +141,7 @@
/*
* One time encoding of the schema name as a Unicode sort key.
*/
- schemaBytes = KeyBuilder.asSortKey(name);
+ schemaBytes = asSortKey(name);
}
}
@@ -501,5 +501,51 @@
+ ",primaryKeyType=" + getPrimaryKeyType() + "}";
}
+
+ /**
+ * Used for historical compatibility to unbox an application key (convert it
+ * to an unsigned byte[]).
+ */
+ static private final IKeyBuilder _keyBuilder = KeyBuilder.newUnicodeInstance();
+
+ /**
+ * Utility method for historical compatibility converts an application key
+ * to a sort key (an unsigned byte[] that imposes the same sort order).
+ * <p>
+ * Note: This method is thread-safe.
+ *
+ * @param val
+ * An application key.
+ *
+ * @return The unsigned byte[] equivalent of that key. This will be
+ * <code>null</code> iff the <i>key</i> is <code>null</code>. If the
+ * <i>key</i> is a byte[], then the byte[] itself will be returned.
+ */
+ private static final byte[] asSortKey(final Object val) {
+
+ if (val == null) {
+
+ return null;
+
+ }
+
+ if (val instanceof byte[]) {
+
+ return (byte[]) val;
+
+ }
+
+ /*
+ * Synchronize on the keyBuilder to avoid concurrent modification of its
+ * state.
+ */
+
+ synchronized (_keyBuilder) {
+
+ return _keyBuilder.getSortKey(val);
+
+ }
+ }
+
}
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractBTreeTestCase.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractBTreeTestCase.java 2010-07-30 22:14:17 UTC (rev 3379)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractBTreeTestCase.java 2010-07-30 22:46:19 UTC (rev 3380)
@@ -49,6 +49,7 @@
import com.bigdata.btree.keys.IKeyBuilder;
import com.bigdata.btree.keys.KV;
import com.bigdata.btree.keys.KeyBuilder;
+import com.bigdata.btree.keys.TestKeyBuilder;
import com.bigdata.btree.raba.IRaba;
import com.bigdata.btree.raba.codec.RandomKeysGenerator;
import com.bigdata.cache.HardReferenceQueue;
@@ -1097,7 +1098,7 @@
assertEquals("#entries",i,btree.nentries);
- final byte[] key = KeyBuilder.asSortKey(ikey);
+ final byte[] key = TestKeyBuilder.asSortKey(ikey);
assertNull(btree.lookup(key));
@@ -1133,7 +1134,7 @@
for( int i=0; i<keys.length; i++ ) {
- byte[] key = KeyBuilder.asSortKey(keys[i]);
+ byte[] key = TestKeyBuilder.asSortKey(keys[i]);
assertEquals(entries[i],btree.lookup(key));
assertEquals(entries[i],btree.remove(key));
@@ -1216,7 +1217,7 @@
assertEquals("#entries",i,btree.nentries);
- final byte[] key = KeyBuilder.asSortKey(ikey);
+ final byte[] key = TestKeyBuilder.asSortKey(ikey);
assertNull(btree.lookup(key));
@@ -1252,7 +1253,7 @@
for( int i=0; i<keys.length; i++ ) {
- final byte[] key = KeyBuilder.asSortKey(keys[i]);
+ final byte[] key = TestKeyBuilder.asSortKey(keys[i]);
assertEquals(entries[i],btree.lookup(key));
assertEquals(entries[i],btree.remove(key));
@@ -1497,7 +1498,7 @@
assertEquals("#entries", i, btree.nentries);
- final byte[] key = KeyBuilder.asSortKey(ikey);
+ final byte[] key = TestKeyBuilder.asSortKey(ikey);
assertNull(btree.lookup(key));
@@ -1687,7 +1688,7 @@
assertEquals("#entries",i,btree.nentries);
- final byte[] key = KeyBuilder.asSortKey(ikey);
+ final byte[] key = TestKeyBuilder.asSortKey(ikey);
assertNull(btree.lookup(key));
@@ -1774,7 +1775,7 @@
final Integer ikey = keys[index];
- final byte[] key = KeyBuilder.asSortKey(ikey);
+ final byte[] key = TestKeyBuilder.asSortKey(ikey);
final SimpleEntry val = vals[index];
@@ -1816,7 +1817,7 @@
Map.Entry<Integer,SimpleEntry> entry = itr.next();
- final byte[] tmp = KeyBuilder.asSortKey(entry.getKey());
+ final byte[] tmp = TestKeyBuilder.asSortKey(entry.getKey());
assertEquals("lookup(" + entry.getKey() + ")", entry
.getValue(), btree.lookup(tmp));
@@ -1855,7 +1856,7 @@
for( int i=0; i<nkeys; i++ ) {
- keys[i] = KeyBuilder.asSortKey(i+1); // Note: this produces dense keys with origin ONE(1).
+ keys[i] = TestKeyBuilder.asSortKey(i+1); // Note: this produces dense keys with origin ONE(1).
vals[i] = new SimpleEntry();
@@ -2597,7 +2598,7 @@
for (int i = 0; i < N; i++) {
// @todo param governs chance of a key collision and maximum #of distinct keys.
- final byte[] key = KeyBuilder.asSortKey(r.nextInt(100000));
+ final byte[] key = TestKeyBuilder.asSortKey(r.nextInt(100000));
// Note: #of bytes effects very little that we want to test so we keep it small.
final byte[] val = new byte[4];
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractTupleCursorTestCase.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractTupleCursorTestCase.java 2010-07-30 22:14:17 UTC (rev 3379)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractTupleCursorTestCase.java 2010-07-30 22:46:19 UTC (rev 3380)
@@ -33,7 +33,7 @@
import junit.framework.TestCase2;
-import com.bigdata.btree.keys.KeyBuilder;
+import com.bigdata.btree.keys.TestKeyBuilder;
import com.bigdata.btree.raba.ReadOnlyKeysRaba;
import com.bigdata.rawstore.SimpleMemoryRawStore;
@@ -287,65 +287,65 @@
// seek to a probe key that does not exist.
assertEquals(null, cursor.seek(29));
assertEquals(null, cursor.tuple());
- assertEquals(KeyBuilder.asSortKey(29),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(29),cursor.currentKey());
assertTrue(cursor.hasNext());
assertEquals(new TestTuple<String>(30, "James"), cursor.next());
assertEquals(new TestTuple<String>(30, "James"), cursor.tuple());
- assertEquals(KeyBuilder.asSortKey(30),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(30),cursor.currentKey());
assertFalse(cursor.hasNext());
assertTrue(cursor.hasPrior());
assertEquals(new TestTuple<String>(20, "Mike"), cursor.prior());
assertEquals(new TestTuple<String>(20, "Mike"), cursor.tuple());
- assertEquals(KeyBuilder.asSortKey(20),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(20),cursor.currentKey());
// seek to a probe key that does not exist.
assertEquals(null, cursor.seek(9));
assertEquals(null, cursor.tuple());
- assertEquals(KeyBuilder.asSortKey(9),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(9),cursor.currentKey());
assertTrue(cursor.hasNext());
assertEquals(new TestTuple<String>(10, "Bryan"), cursor.next());
assertEquals(new TestTuple<String>(10, "Bryan"), cursor.tuple());
- assertEquals(KeyBuilder.asSortKey(10),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(10),cursor.currentKey());
assertFalse(cursor.hasPrior());
assertTrue(cursor.hasNext());
assertEquals(new TestTuple<String>(20, "Mike"), cursor.next());
- assertEquals(KeyBuilder.asSortKey(20),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(20),cursor.currentKey());
// seek to a probe key that does not exist and scan forward.
assertEquals(null, cursor.seek(19));
assertEquals(null, cursor.tuple());
- assertEquals(KeyBuilder.asSortKey(19),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(19),cursor.currentKey());
assertTrue(cursor.hasNext());
assertEquals(new TestTuple<String>(20, "Mike"), cursor.next());
assertEquals(new TestTuple<String>(20, "Mike"), cursor.tuple());
- assertEquals(KeyBuilder.asSortKey(20),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(20),cursor.currentKey());
assertTrue(cursor.hasNext());
assertEquals(new TestTuple<String>(30, "James"), cursor.next());
assertEquals(new TestTuple<String>(30, "James"), cursor.tuple());
- assertEquals(KeyBuilder.asSortKey(30),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(30),cursor.currentKey());
// seek to a probe key that does not exist and scan backward.
assertEquals(null, cursor.seek(19));
assertEquals(null, cursor.tuple());
- assertEquals(KeyBuilder.asSortKey(19),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(19),cursor.currentKey());
assertTrue(cursor.hasPrior());
assertEquals(new TestTuple<String>(10, "Bryan"), cursor.prior());
assertEquals(new TestTuple<String>(10, "Bryan"), cursor.tuple());
- assertEquals(KeyBuilder.asSortKey(10),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(10),cursor.currentKey());
assertFalse(cursor.hasPrior());
// seek to a probe key that does not exist (after all valid tuples).
assertEquals(null, cursor.seek(31));
assertEquals(null, cursor.tuple());
assertTrue(cursor.isCursorPositionDefined());
- assertEquals(KeyBuilder.asSortKey(31),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(31),cursor.currentKey());
assertFalse(cursor.hasNext());
// seek to a probe key that does not exist (after all valid tuples).
assertEquals(null, cursor.seek(31));
assertEquals(null, cursor.tuple());
assertTrue(cursor.isCursorPositionDefined());
- assertEquals(KeyBuilder.asSortKey(31),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(31),cursor.currentKey());
assertTrue(cursor.hasPrior());
assertEquals(new TestTuple<String>(30, "James"), cursor.prior());
@@ -369,9 +369,9 @@
*/
{
- final byte[] fromKey = KeyBuilder.asSortKey(10);
+ final byte[] fromKey = TestKeyBuilder.asSortKey(10);
- final byte[] toKey = KeyBuilder.asSortKey(20);
+ final byte[] toKey = TestKeyBuilder.asSortKey(20);
final ITupleCursor2<String> cursor = newCursor(btree,
IRangeQuery.DEFAULT, fromKey, toKey);
@@ -412,9 +412,9 @@
*/
{
- final byte[] fromKey = KeyBuilder.asSortKey(20);
+ final byte[] fromKey = TestKeyBuilder.asSortKey(20);
- final byte[] toKey = KeyBuilder.asSortKey(30);
+ final byte[] toKey = TestKeyBuilder.asSortKey(30);
final ITupleCursor2<String> cursor = newCursor(btree,
IRangeQuery.DEFAULT, fromKey, toKey);
@@ -458,9 +458,9 @@
*/
{
- final byte[] fromKey = KeyBuilder.asSortKey(10);
+ final byte[] fromKey = TestKeyBuilder.asSortKey(10);
- final byte[] toKey = KeyBuilder.asSortKey(19);
+ final byte[] toKey = TestKeyBuilder.asSortKey(19);
final ITupleCursor2<String> cursor = newCursor(btree,
IRangeQuery.DEFAULT, fromKey, toKey);
@@ -471,7 +471,7 @@
// assertEquals(KeyBuilder.asSortKey(19),cursor.currentKey());
assertTrue(cursor.hasPrior());
assertEquals(new TestTuple<String>(10, "Bryan"), cursor.prior());
- assertEquals(KeyBuilder.asSortKey(10),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(10),cursor.currentKey());
assertFalse(cursor.hasPrior());
}
@@ -481,19 +481,19 @@
*/
{
- final byte[] fromKey = KeyBuilder.asSortKey(10);
+ final byte[] fromKey = TestKeyBuilder.asSortKey(10);
- final byte[] toKey = KeyBuilder.asSortKey(29);
+ final byte[] toKey = TestKeyBuilder.asSortKey(29);
final ITupleCursor2<String> cursor = newCursor(btree,
IRangeQuery.DEFAULT, fromKey, toKey);
assertTrue(cursor.hasPrior());
assertEquals(new TestTuple<String>(20, "Mike"), cursor.prior());
- assertEquals(KeyBuilder.asSortKey(20),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(20),cursor.currentKey());
assertTrue(cursor.hasPrior());
assertEquals(new TestTuple<String>(10, "Bryan"), cursor.prior());
- assertEquals(KeyBuilder.asSortKey(10),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(10),cursor.currentKey());
assertFalse(cursor.hasPrior());
}
@@ -503,16 +503,16 @@
*/
{
- final byte[] fromKey = KeyBuilder.asSortKey(10);
+ final byte[] fromKey = TestKeyBuilder.asSortKey(10);
- final byte[] toKey = KeyBuilder.asSortKey(11);
+ final byte[] toKey = TestKeyBuilder.asSortKey(11);
final ITupleCursor2<String> cursor = newCursor(btree,
IRangeQuery.DEFAULT, fromKey, toKey);
assertTrue(cursor.hasPrior());
assertEquals(new TestTuple<String>(10, "Bryan"), cursor.prior());
- assertEquals(KeyBuilder.asSortKey(10),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(10),cursor.currentKey());
assertFalse(cursor.hasPrior());
}
@@ -574,15 +574,15 @@
assertEquals("ntuples", 5, btree.getEntryCount());
// The separator key is (30).
- assertEquals(KeyBuilder.asSortKey(30), ((Node) btree.getRoot())
+ assertEquals(TestKeyBuilder.asSortKey(30), ((Node) btree.getRoot())
.getKeys().get(0));
// Verify the expected keys in the 1st leaf.
AbstractBTreeTestCase.assertKeys(
//
new ReadOnlyKeysRaba(new byte[][] {//
- KeyBuilder.asSortKey(10), //
- KeyBuilder.asSortKey(20), //
+ TestKeyBuilder.asSortKey(10), //
+ TestKeyBuilder.asSortKey(20), //
}),//
((Node) btree.getRoot()).getChild(0/* 1st leaf */).getKeys());
@@ -590,9 +590,9 @@
AbstractBTreeTestCase.assertKeys(
//
new ReadOnlyKeysRaba(new byte[][] {//
- KeyBuilder.asSortKey(30), //
- KeyBuilder.asSortKey(40), //
- KeyBuilder.asSortKey(50),//
+ TestKeyBuilder.asSortKey(30), //
+ TestKeyBuilder.asSortKey(40), //
+ TestKeyBuilder.asSortKey(50),//
}),//
((Node) btree.getRoot()).getChild(1/* 2nd leaf */).getKeys());
@@ -627,16 +627,16 @@
*/
{
- final byte[] fromKey = KeyBuilder.asSortKey(10);
+ final byte[] fromKey = TestKeyBuilder.asSortKey(10);
- final byte[] toKey = KeyBuilder.asSortKey(30);
+ final byte[] toKey = TestKeyBuilder.asSortKey(30);
final ITupleCursor2<String> cursor = newCursor(btree,
IRangeQuery.DEFAULT, fromKey, toKey);
assertTrue(cursor.hasPrior());
assertEquals(new TestTuple<String>(20, "Mike"), cursor.prior());
- assertEquals(KeyBuilder.asSortKey(20),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(20),cursor.currentKey());
}
@@ -647,16 +647,16 @@
*/
{
- final byte[] fromKey = KeyBuilder.asSortKey(0);
+ final byte[] fromKey = TestKeyBuilder.asSortKey(0);
- final byte[] toKey = KeyBuilder.asSortKey(19);
+ final byte[] toKey = TestKeyBuilder.asSortKey(19);
final ITupleCursor2<String> cursor = newCursor(btree,
IRangeQuery.DEFAULT, fromKey, toKey);
assertTrue(cursor.hasPrior());
assertEquals(new TestTuple<String>(10, "Bryan"), cursor.prior());
- assertEquals(KeyBuilder.asSortKey(10),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(10),cursor.currentKey());
assertFalse(cursor.hasPrior());
}
@@ -668,9 +668,9 @@
*/
{
- final byte[] fromKey = KeyBuilder.asSortKey(0);
+ final byte[] fromKey = TestKeyBuilder.asSortKey(0);
- final byte[] toKey = KeyBuilder.asSortKey(9);
+ final byte[] toKey = TestKeyBuilder.asSortKey(9);
final ITupleCursor2<String> cursor = newCursor(btree,
IRangeQuery.DEFAULT, fromKey, toKey);
@@ -698,7 +698,7 @@
* Verify that the separatorKey in the parent is the first tuple we
* expect to find in the 2nd leaf.
*/
- assertEquals(KeyBuilder.asSortKey(30), ((Node) btree.getRoot())
+ assertEquals(TestKeyBuilder.asSortKey(30), ((Node) btree.getRoot())
.getKeys().get(0));
/*
@@ -711,29 +711,29 @@
// Remove the first tuple in the 2nd leaf.
btree.remove(30);
// The separator key has not been changed.
- assertEquals(((Node) btree.getRoot()).getKeys().get(0), KeyBuilder
+ assertEquals(((Node) btree.getRoot()).getKeys().get(0), TestKeyBuilder
.asSortKey(30));
// The #of leaves has not been changed.
assertEquals(2, btree.getLeafCount());
// Verify the expected keys in the 2nd leaf.
AbstractBTreeTestCase.assertKeys(//
new ReadOnlyKeysRaba(new byte[][]{//
- KeyBuilder.asSortKey(40),//
- KeyBuilder.asSortKey(50),//
+ TestKeyBuilder.asSortKey(40),//
+ TestKeyBuilder.asSortKey(50),//
}),//
((Node) btree.getRoot()).getChild(1/*2nd leaf*/).getKeys());
- final byte[] fromKey = KeyBuilder.asSortKey(10);
+ final byte[] fromKey = TestKeyBuilder.asSortKey(10);
// search for the tuple we just deleted from the 2nd leaf.
- final byte[] toKey = KeyBuilder.asSortKey(30);
+ final byte[] toKey = TestKeyBuilder.asSortKey(30);
final ITupleCursor2<String> cursor = newCursor(btree,
IRangeQuery.DEFAULT, fromKey, toKey);
assertTrue(cursor.hasPrior());
assertEquals(new TestTuple<String>(20, "Mike"), cursor.prior());
- assertEquals(KeyBuilder.asSortKey(20), cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(20), cursor.currentKey());
assertTrue(cursor.hasPrior());
}
@@ -862,9 +862,9 @@
*/
{
- final byte[] fromKey = KeyBuilder.asSortKey(2);
+ final byte[] fromKey = TestKeyBuilder.asSortKey(2);
- final byte[] toKey = KeyBuilder.asSortKey(7);
+ final byte[] toKey = TestKeyBuilder.asSortKey(7);
// first()
{
@@ -1107,7 +1107,7 @@
assertNull(cursor.seek(1));
- assertEquals(KeyBuilder.asSortKey(1), cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(1), cursor.currentKey());
assertFalse(cursor.hasPrior());
@@ -1141,9 +1141,9 @@
*/
{
- final byte[] fromKey = KeyBuilder.asSortKey(5);
+ final byte[] fromKey = TestKeyBuilder.asSortKey(5);
- final byte[] toKey = KeyBuilder.asSortKey(9);
+ final byte[] toKey = TestKeyBuilder.asSortKey(9);
// first()
{
@@ -1237,7 +1237,7 @@
assertNull(cursor.seek(7));
- assertEquals(KeyBuilder.asSortKey(7), cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(7), cursor.currentKey());
assertFalse(cursor.hasPrior());
@@ -1254,9 +1254,9 @@
*/
{
- final byte[] fromKey = KeyBuilder.asSortKey(15);
+ final byte[] fromKey = TestKeyBuilder.asSortKey(15);
- final byte[] toKey = KeyBuilder.asSortKey(19);
+ final byte[] toKey = TestKeyBuilder.asSortKey(19);
// first()
{
@@ -1338,7 +1338,7 @@
assertNull(cursor.seek(17));
- assertEquals(KeyBuilder.asSortKey(17), cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(17), cursor.currentKey());
assertFalse(cursor.hasPrior());
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestBTreeLeafCursors.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestBTreeLeafCursors.java 2010-07-30 22:14:17 UTC (rev 3379)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestBTreeLeafCursors.java 2010-07-30 22:46:19 UTC (rev 3380)
@@ -33,7 +33,7 @@
import junit.framework.TestCase2;
import com.bigdata.btree.BTree.Stack;
-import com.bigdata.btree.keys.KeyBuilder;
+import com.bigdata.btree.keys.TestKeyBuilder;
import com.bigdata.rawstore.SimpleMemoryRawStore;
/**
@@ -125,56 +125,56 @@
ILeafCursor<Leaf> cursor = btree.newLeafCursor(SeekEnum.First);
// verify first leaf since that is where we positioned the cursor.
- assertEquals(KeyBuilder.asSortKey(1), cursor.leaf().getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(1), cursor.leaf().getKeys().get(0));
// first().
- assertEquals(KeyBuilder.asSortKey(1), cursor.first().getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(1), cursor.first().getKeys().get(0));
// last().
- assertEquals(KeyBuilder.asSortKey(9), cursor.last().getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(9), cursor.last().getKeys().get(0));
}
public void test_seek() {
- ILeafCursor<Leaf> cursor = btree.newLeafCursor(KeyBuilder.asSortKey(5));
+ ILeafCursor<Leaf> cursor = btree.newLeafCursor(TestKeyBuilder.asSortKey(5));
// verify initial seek.
- assertEquals(KeyBuilder.asSortKey(5), cursor.leaf().getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(5), cursor.leaf().getKeys().get(0));
// verify seek to each key found in the B+Tree.
- assertEquals(KeyBuilder.asSortKey(1), cursor.seek(
- KeyBuilder.asSortKey(1)).getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(1), cursor.seek(
+ TestKeyBuilder.asSortKey(1)).getKeys().get(0));
- assertEquals(KeyBuilder.asSortKey(1), cursor.seek(
- KeyBuilder.asSortKey(2)).getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(1), cursor.seek(
+ TestKeyBuilder.asSortKey(2)).getKeys().get(0));
- assertEquals(KeyBuilder.asSortKey(3), cursor.seek(
- KeyBuilder.asSortKey(3)).getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(3), cursor.seek(
+ TestKeyBuilder.asSortKey(3)).getKeys().get(0));
- assertEquals(KeyBuilder.asSortKey(3), cursor.seek(
- KeyBuilder.asSortKey(4)).getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(3), cursor.seek(
+ TestKeyBuilder.asSortKey(4)).getKeys().get(0));
- assertEquals(KeyBuilder.asSortKey(5), cursor.seek(
- KeyBuilder.asSortKey(5)).getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(5), cursor.seek(
+ TestKeyBuilder.asSortKey(5)).getKeys().get(0));
- assertEquals(KeyBuilder.asSortKey(5), cursor.seek(
- KeyBuilder.asSortKey(6)).getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(5), cursor.seek(
+ TestKeyBuilder.asSortKey(6)).getKeys().get(0));
- assertEquals(KeyBuilder.asSortKey(7), cursor.seek(
- KeyBuilder.asSortKey(7)).getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(7), cursor.seek(
+ TestKeyBuilder.asSortKey(7)).getKeys().get(0));
- assertEquals(KeyBuilder.asSortKey(7), cursor.seek(
- KeyBuilder.asSortKey(8)).getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(7), cursor.seek(
+ TestKeyBuilder.asSortKey(8)).getKeys().get(0));
- assertEquals(KeyBuilder.asSortKey(9), cursor.seek(
- KeyBuilder.asSortKey(9)).getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(9), cursor.seek(
+ TestKeyBuilder.asSortKey(9)).getKeys().get(0));
- assertEquals(KeyBuilder.asSortKey(9), cursor.seek(
- KeyBuilder.asSortKey(10)).getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(9), cursor.seek(
+ TestKeyBuilder.asSortKey(10)).getKeys().get(0));
// verify seek to key that would be in the last leaf but is not actually in the B+Tree.
- assertEquals(KeyBuilder.asSortKey(9),cursor.seek(KeyBuilder.asSortKey(12)).getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(9),cursor.seek(TestKeyBuilder.asSortKey(12)).getKeys().get(0));
}
@@ -184,19 +184,19 @@
ILeafCursor<Leaf> cursor = btree.newLeafCursor(SeekEnum.First);
// verify first leaf since that is where we positioned the cursor.
- assertEquals(KeyBuilder.asSortKey(1), cursor.leaf().getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(1), cursor.leaf().getKeys().get(0));
// next().
- assertEquals(KeyBuilder.asSortKey(3), cursor.next().getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(3), cursor.next().getKeys().get(0));
// next().
- assertEquals(KeyBuilder.asSortKey(5), cursor.next().getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(5), cursor.next().getKeys().get(0));
// next().
- assertEquals(KeyBuilder.asSortKey(7), cursor.next().getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(7), cursor.next().getKeys().get(0));
// next().
- assertEquals(KeyBuilder.asSortKey(9), cursor.next().getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(9), cursor.next().getKeys().get(0));
}
@@ -205,19 +205,19 @@
ILeafCursor<Leaf> cursor = btree.newLeafCursor(SeekEnum.Last);
// verify last leaf since that is where we positioned the cursor.
- assertEquals(KeyBuilder.asSortKey(9), cursor.leaf().getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(9), cursor.leaf().getKeys().get(0));
// next().
- assertEquals(KeyBuilder.asSortKey(7), cursor.prior().getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(7), cursor.prior().getKeys().get(0));
// next().
- assertEquals(KeyBuilder.asSortKey(5), cursor.prior().getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(5), cursor.prior().getKeys().get(0));
// next().
- assertEquals(KeyBuilder.asSortKey(3), cursor.prior().getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(3), cursor.prior().getKeys().get(0));
// next().
- assertEquals(KeyBuilder.asSortKey(1), cursor.prior().getKeys().get(0));
+ assertEquals(TestKeyBuilder.asSortKey(1), cursor.prior().getKeys().get(0));
}
@@ -247,7 +247,7 @@
for (int i = 1; i <= 10; i++) {
- btree.insert(KeyBuilder.asSortKey(i), "v"+i);
+ btree.insert(TestKeyBuilder.asSortKey(i), "v"+i);
}
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestBigdataMap.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestBigdataMap.java 2010-07-30 22:14:17 UTC (rev 3379)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestBigdataMap.java 2010-07-30 22:46:19 UTC (rev 3380)
@@ -39,7 +39,7 @@
import com.bigdata.btree.keys.DefaultKeyBuilderFactory;
import com.bigdata.btree.keys.IKeyBuilderFactory;
-import com.bigdata.btree.keys.KeyBuilder;
+import com.bigdata.btree.keys.TestKeyBuilder;
import com.bigdata.io.SerializerUtil;
import com.bigdata.rawstore.SimpleMemoryRawStore;
@@ -163,7 +163,7 @@
* Handles {@link String} keys and values and makes the keys available for
* {@link BigdataMap} and {@link BigdataSet} (under the assumption that the
* key and the value are the same!). The actual index order is governed by
- * {@link KeyBuilder#asSortKey(Object)}.
+ * {@link TestKeyBuilder#asSortKey(Object)}.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestChunkedIterators.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestChunkedIterators.java 2010-07-30 22:14:17 UTC (rev 3379)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestChunkedIterators.java 2010-07-30 22:46:19 UTC (rev 3380)
@@ -36,6 +36,7 @@
import com.bigdata.btree.filter.TupleFilter;
import com.bigdata.btree.keys.DefaultKeyBuilderFactory;
import com.bigdata.btree.keys.KeyBuilder;
+import com.bigdata.btree.keys.TestKeyBuilder;
import com.bigdata.io.SerializerUtil;
import com.bigdata.rawstore.IBlock;
import com.bigdata.rawstore.SimpleMemoryRawStore;
@@ -109,7 +110,7 @@
for(int i=0; i<nentries; i++) {
- keys[i] = KeyBuilder.asSortKey(i);
+ keys[i] = TestKeyBuilder.asSortKey(i);
vals[i] = new byte[4];
@@ -203,7 +204,7 @@
for(int i=0; i<nentries; i++) {
- keys[i] = KeyBuilder.asSortKey(i);
+ keys[i] = TestKeyBuilder.asSortKey(i);
vals[i] = new byte[4];
@@ -337,7 +338,7 @@
for(int i=0; i<nentries; i++) {
- keys[i] = KeyBuilder.asSortKey(i);
+ keys[i] = TestKeyBuilder.asSortKey(i);
vals[i] = new byte[4];
@@ -425,7 +426,7 @@
for(int i=0; i<nentries; i++) {
- keys[i] = KeyBuilder.asSortKey(i);
+ keys[i] = TestKeyBuilder.asSortKey(i);
vals[i] = new byte[4];
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestCopyOnWrite.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestCopyOnWrite.java 2010-07-30 22:14:17 UTC (rev 3379)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestCopyOnWrite.java 2010-07-30 22:46:19 UTC (rev 3380)
@@ -29,7 +29,7 @@
import org.apache.log4j.Level;
-import com.bigdata.btree.keys.KeyBuilder;
+import com.bigdata.btree.keys.TestKeyBuilder;
/**
* Test suite for copy-on-write semantics. Among other things the tests in this
@@ -79,12 +79,12 @@
SimpleEntry v9 = new SimpleEntry(9);
// fill up the root leaf.
- btree.insert(KeyBuilder.asSortKey(3), v3);
- btree.insert(KeyBuilder.asSortKey(5), v5);
- btree.insert(KeyBuilder.asSortKey(7), v7);
+ btree.insert(TestKeyBuilder.asSortKey(3), v3);
+ btree.insert(TestKeyBuilder.asSortKey(5), v5);
+ btree.insert(TestKeyBuilder.asSortKey(7), v7);
// split the root leaf.
- btree.insert(KeyBuilder.asSortKey(9), v9);
+ btree.insert(TestKeyBuilder.asSortKey(9), v9);
final Node c = (Node) btree.root;
assertKeys(new int[]{7},c);
assertEquals(a,c.getChild(0));
@@ -100,8 +100,8 @@
* split another leaf so that there are now three children to visit. at
* this point the root is full.
*/
- btree.insert(KeyBuilder.asSortKey(1), v1);
- btree.insert(KeyBuilder.asSortKey(2), v2);
+ btree.insert(TestKeyBuilder.asSortKey(1), v1);
+ btree.insert(TestKeyBuilder.asSortKey(2), v2);
assertKeys(new int[]{3,7},c);
assertEquals(a,c.getChild(0));
Leaf d = (Leaf)c.getChild(1);
@@ -141,7 +141,7 @@
* triggers copy-on-write for (a). (a1) is dirty as a post-condition.
* (d) is deleted as a post-condition.
*/
- assertEquals(v1,btree.remove(KeyBuilder.asSortKey(1)));
+ assertEquals(v1,btree.remove(TestKeyBuilder.asSortKey(1)));
assertKeys(new int[]{7},c);
assertNotSame(a,c.getChild(0));
final Leaf a1 = (Leaf)c.getChild(0);
@@ -160,7 +160,7 @@
* insert a key that will go into (b). since (b) is immutable this
* triggers copy-on-write.
*/
- btree.insert(KeyBuilder.asSortKey(8),v8);
+ btree.insert(TestKeyBuilder.asSortKey(8),v8);
assertKeys(new int[]{7},c);
assertEquals(a1,c.getChild(0));
assertNotSame(b,c.getChild(1));
@@ -194,7 +194,7 @@
* (b1) is clean, so it is stolen by setting its parent reference
* to the new (c1).
*/
- assertEquals(v2,btree.remove(KeyBuilder.asSortKey(2)));
+ assertEquals(v2,btree.remove(TestKeyBuilder.asSortKey(2)));
assertNotSame(c,btree.root);
final Node c1 = (Node)btree.root;
assertKeys(new int[]{7},c1);
@@ -245,14 +245,14 @@
* copy-on-write. We verify that the root leaf reference is changed.
*/
assertEquals(a,btree.root);
- btree.insert(KeyBuilder.asSortKey(3), v3);
+ btree.insert(TestKeyBuilder.asSortKey(3), v3);
assertNotSame(a,btree.root);
a = (Leaf)btree.root; // new reference for the root leaf.
- btree.insert(KeyBuilder.asSortKey(5), v5);
- btree.insert(KeyBuilder.asSortKey(7), v7);
+ btree.insert(TestKeyBuilder.asSortKey(5), v5);
+ btree.insert(TestKeyBuilder.asSortKey(7), v7);
// split the root leaf.
- btree.insert(KeyBuilder.asSortKey(9), v9);
+ btree.insert(TestKeyBuilder.asSortKey(9), v9);
Node c = (Node) btree.root;
assertKeys(new int[]{7},c);
assertEquals(a,c.getChild(0));
@@ -289,7 +289,7 @@
assertTrue(a.isPersistent());
assertTrue(b.isPersistent());
assertTrue(c.isPersistent());
- btree.insert(KeyBuilder.asSortKey(1), v1); // triggers copy on write for (a) and (c).
+ btree.insert(TestKeyBuilder.asSortKey(1), v1); // triggers copy on write for (a) and (c).
assertNotSame(c,btree.root);
c = (Node)btree.root;
assertNotSame(a,c.getChild(0));
@@ -300,7 +300,7 @@
assertTrue(b.isPersistent());
assertFalse(c.isPersistent());
// insert more until we split another leaf.
- btree.insert(KeyBuilder.asSortKey(2), v2);
+ btree.insert(TestKeyBuilder.asSortKey(2), v2);
assertKeys(new int[]{3,7},c);
assertEquals(a,c.getChild(0));
Leaf d = (Leaf)c.getChild(1); // the new leaf (d).
@@ -323,8 +323,8 @@
* cause another leaf (d) to split, forcing the split to propagate to and
* split the root and the tree to increase in height.
*/
- btree.insert(KeyBuilder.asSortKey(4), v4);
- btree.insert(KeyBuilder.asSortKey(6), v6);
+ btree.insert(TestKeyBuilder.asSortKey(4), v4);
+ btree.insert(TestKeyBuilder.asSortKey(6), v6);
// btree.dump(Level.DEBUG,System.err);
assertNotSame(c,btree.root);
final Node g = (Node)btree.root;
@@ -365,7 +365,7 @@
* the following are cloned: d, c, g.
* the following clean children are stolen: e, b (by the new root c).
*/
- assertEquals(v4,btree.remove(KeyBuilder.asSortKey(4)));
+ assertEquals(v4,btree.remove(TestKeyBuilder.asSortKey(4)));
assertNotSame(g,btree.root);
assertNotSame(c,btree.root);
c = (Node) btree.root;
@@ -393,7 +393,7 @@
* remove a key (7) from a leaf (b) forcing two leaves (b,e) to join
* into (b)
*/
- assertEquals(v7,btree.remove(KeyBuilder.asSortKey(7)));
+ assertEquals(v7,btree.remove(TestKeyBuilder.asSortKey(7)));
btree.dump(Level.DEBUG,System.err);
assertKeys(new int[]{5},c);
assertEquals(d,c.getChild(0));
@@ -421,16 +421,16 @@
assertEquals(c,btree.root);
assertEquals(d,c.getChild(0));
assertEquals(b,c.getChild(1));
- assertEquals(v3, btree.remove(KeyBuilder.asSortKey(3))); // remove from (d)
+ assertEquals(v3, btree.remove(TestKeyBuilder.asSortKey(3))); // remove from (d)
assertNotSame(c,btree.root); // c was cloned.
c = (Node) btree.root;
assertNotSame(d,c.getChild(0));
d = (Leaf)c.getChild(0); // d was cloned.
assertEquals(b,c.getChild(1));
- assertEquals(v5,btree.remove(KeyBuilder.asSortKey(5))); // remove from (b)
+ assertEquals(v5,btree.remove(TestKeyBuilder.asSortKey(5))); // remove from (b)
assertNotSame(b,c.getChild(1));
b = (Leaf)c.getChild(1); // b was cloned.
- assertEquals(v6,btree.remove(KeyBuilder.asSortKey(6))); // remove from (b)
+ assertEquals(v6,btree.remove(TestKeyBuilder.asSortKey(6))); // remove from (b)
assertKeys(new int[]{1,2,9},b);
assertValues(new Object[]{v1,v2,v9}, b);
assertTrue(d.isDeleted());
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestDirtyIterators.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestDirtyIterators.java 2010-07-30 22:14:17 UTC (rev 3379)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestDirtyIterators.java 2010-07-30 22:46:19 UTC (rev 3380)
@@ -29,7 +29,7 @@
import org.apache.log4j.Level;
-import com.bigdata.btree.keys.KeyBuilder;
+import com.bigdata.btree.keys.TestKeyBuilder;
/**
* Test suite for iterators that visit only dirty nodes or leaves. This test
@@ -82,12 +82,12 @@
SimpleEntry v9 = new SimpleEntry(9);
// fill up the root leaf.
- btree.insert(KeyBuilder.asSortKey(3), v3);
- btree.insert(KeyBuilder.asSortKey(5), v5);
- btree.insert(KeyBuilder.asSortKey(7), v7);
+ btree.insert(TestKeyBuilder.asSortKey(3), v3);
+ btree.insert(TestKeyBuilder.asSortKey(5), v5);
+ btree.insert(TestKeyBuilder.asSortKey(7), v7);
// split the root leaf.
- btree.insert(KeyBuilder.asSortKey(9), v9);
+ btree.insert(TestKeyBuilder.asSortKey(9), v9);
final Node c = (Node) btree.root;
assertKeys(new int[]{7},c);
assertEquals(a,c.getChild(0));
@@ -109,8 +109,8 @@
* split another leaf so that there are now three children to visit. at
* this point the root is full.
*/
- btree.insert(KeyBuilder.asSortKey(1), v1);
- btree.insert(KeyBuilder.asSortKey(2), v2);
+ btree.insert(TestKeyBuilder.asSortKey(1), v1);
+ btree.insert(TestKeyBuilder.asSortKey(2), v2);
assertKeys(new int[]{3,7},c);
assertEquals(a,c.getChild(0));
Leaf d = (Leaf)c.getChild(1);
@@ -135,7 +135,7 @@
* remove a key from a leaf forcing two leaves to join and verify the
* visitation order.
*/
- assertEquals(v1,btree.remove(KeyBuilder.asSortKey(1)));
+ assertEquals(v1,btree.remove(TestKeyBuilder.asSortKey(1)));
assertKeys(new int[]{7},c);
assertEquals(a,c.getChild(0));
assertEquals(b,c.getChild(1));
@@ -187,12 +187,12 @@
SimpleEntry v9 = new SimpleEntry(9);
// fill up the root leaf.
- btree.insert(KeyBuilder.asSortKey(3), v3);
- btree.insert(KeyBuilder.asSortKey(5), v5);
- btree.insert(KeyBuilder.asSortKey(7), v7);
+ btree.insert(TestKeyBuilder.asSortKey(3), v3);
+ btree.insert(TestKeyBuilder.asSortKey(5), v5);
+ btree.insert(TestKeyBuilder.asSortKey(7), v7);
// split the root leaf.
- btree.insert(KeyBuilder.asSortKey(9), v9);
+ btree.insert(TestKeyBuilder.asSortKey(9), v9);
final Node c = (Node) btree.root;
assertKeys(new int[]{7},c);
assertEquals(a,c.getChild(0));
@@ -212,8 +212,8 @@
* split another leaf so that there are now three children to visit. at
* this point the root is full.
*/
- btree.insert(KeyBuilder.asSortKey(1), v1);
- btree.insert(KeyBuilder.asSortKey(2), v2);
+ btree.insert(TestKeyBuilder.asSortKey(1), v1);
+ btree.insert(TestKeyBuilder.asSortKey(2), v2);
assertKeys(new int[]{3,7},c);
assertEquals(a,c.getChild(0));
Leaf d = (Leaf)c.getChild(1);
@@ -258,7 +258,7 @@
* visitation order. this triggers copy-on-write for (a) and (a) is
* dirty as a post-condition.
*/
- assertEquals(v1,btree.remove(KeyBuilder.asSortKey(1)));
+ assertEquals(v1,btree.remove(TestKeyBuilder.asSortKey(1)));
assertKeys(new int[]{7},c);
assertNotSame(a,c.getChild(0));
Leaf a1 = (Leaf)c.getChild(0);
@@ -279,7 +279,7 @@
* insert a key that will go into (b). since (b) is immutable this
* triggers copy-on-write.
*/
- btree.insert(KeyBuilder.asSortKey(8),v8);
+ btree.insert(TestKeyBuilder.asSortKey(8),v8);
assertKeys(new int[]{7},c);
assertEquals(a1,c.getChild(0));
assertNotSame(b,c.getChild(1));
@@ -313,7 +313,7 @@
* remove a key from (a1). since (a1) is immutable this triggers
* copy-on-write. since the root is immtuable, it is also copied.
*/
- assertEquals(v2,btree.remove(KeyBuilder.asSortKey(2)));
+ assertEquals(v2,btree.remove(TestKeyBuilder.asSortKey(2)));
assertNotSame(c,btree.root);
Node c1 = (Node)btree.root;
assertKeys(new int[]{7},c1);
@@ -368,12 +368,12 @@
.postOrderNodeIterator(true));
// fill up the root leaf.
- btree.insert(KeyBuilder.asSortKey(3), v3);
- btree.insert(KeyBuilder.asSortKey(5), v5);
- btree.insert(KeyBuilder.asSortKey(7), v7);
+ btree.insert(TestKeyBuilder.asSortKey(3), v3);
+ btree.insert(TestKeyBuilder.asSortKey(5), v5);
+ btree.insert(TestKeyBuilder.asSortKey(7), v7);
// split the root leaf.
- btree.insert(KeyBuilder.asSortKey(9), v9);
+ btree.insert(TestKeyBuilder.asSortKey(9), v9);
final Node c = (Node) btree.root;
assertKeys(new int[]{7},c);
assertEquals(a,c.getChild(0));
@@ -393,8 +393,8 @@
* split another leaf so that there are now three children to visit. at
* this point the root is full.
*/
- btree.insert(KeyBuilder.asSortKey(1), v1);
- btree.insert(KeyBuilder.asSortKey(2), v2);
+ btree.insert(TestKeyBuilder.asSortKey(1), v1);
+ btree.insert(TestKeyBuilder.asSortKey(2), v2);
assertKeys(new int[]{3,7},c);
assertEquals(a,c.getChild(0));
Leaf d = (Leaf)c.getChild(1);
@@ -416,8 +416,8 @@
* cause another leaf (d) to split, forcing the split to propagate to and
* split the root and the tree to increase in height.
*/
- btree.insert(KeyBuilder.asSortKey(4), v4);
- btree.insert(KeyBuilder.asSortKey(6), v6);
+ btree.insert(TestKeyBuilder.asSortKey(4), v4);
+ btree.insert(TestKeyBuilder.asSortKey(6), v6);
// btree.dump(Level.DEBUG,System.err);
assertNotSame(c,btree.root);
final Node g = (Node)btree.root;
@@ -450,7 +450,7 @@
* be deleted. this causes (c,f) to merge as well, which in turn forces
* the root to be replaced by (c).
*/
- assertEquals(v4,btree.remove(KeyBuilder.asSortKey(4)));
+ assertEquals(v4,btree.remove(TestKeyBuilder.asSortKey(4)));
// btree.dump(Level.DEBUG,System.err);
assertKeys(new int[]{5,7},c);
assertEquals(d,c.getChild(0));
@@ -474,7 +474,7 @@
* remove a key (7) from a leaf (b) forcing two leaves to join and
* verify the visitation order.
*/
- assertEquals(v7,btree.remove(KeyBuilder.asSortKey(7)));
+ assertEquals(v7,btree.remove(TestKeyBuilder.asSortKey(7)));
btree.dump(Level.DEBUG,System.err);
assertKeys(new int[]{5},c);
assertEquals(d,c.getChild(0));
@@ -495,9 +495,9 @@
* remove keys from a leaf forcing the remaining two leaves to join and
* verify the visitation order.
*/
- assertEquals(v3,btree.remove(KeyBuilder.asSortKey(3)));
- assertEquals(v5,btree.remove(KeyBuilder.asSortKey(5)));
- assertEquals(v6,btree.remove(KeyBuilder.asSortKey(6)));
+ assertEquals(v3,btree.remove(TestKeyBuilder.asSortKey(3)));
+ assertEquals(v5,btree.remove(TestKeyBuilder.asSortKey(5)));
+ assertEquals(v6,btree.remove(TestKeyBuilder.asSortKey(6)));
assertKeys(new int[]{1,2,9},b);
assertValues(new Object[]{v1,v2,v9}, b);
assertTrue(d.isDeleted());
@@ -560,18 +560,18 @@
* and verify that both iterators now visit the root.
*/
assertEquals(a,btree.root);
- btree.insert(KeyBuilder.asSortKey(3), v3);
+ btree.insert(TestKeyBuilder.asSortKey(3), v3);
assertNotSame(a,btree.root);
a = (Leaf)btree.root; // new reference for the root leaf.
assertSameIterator(new IAbstractNode[] { btree.root }, btree.root
...
[truncated message content] |
|
From: <btm...@us...> - 2010-08-06 16:51:54
|
Revision: 3426
http://bigdata.svn.sourceforge.net/bigdata/?rev=3426&view=rev
Author: btmurphy
Date: 2010-08-06 16:51:44 +0000 (Fri, 06 Aug 2010)
Log Message:
-----------
[branch dev-btm]: CHECKPOINT - changes to allow smart proxy implementation of load balancer to be started by ServicesManagerService or BootManager, new/changed tests to run against either the smart proxy implementation of the load balancer or the purely remote implemenation (LoadBalancerService & LoadBalancerService)
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/RuleStats.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/CompactingMergeTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/DefaultServiceFederationDelegate.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedClient.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/Event.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/HostScore.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IBigdataClient.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IBigdataFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IFederationDelegate.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ILoadBalancerService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IServiceLoadHelper.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/LoadBalancer.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ServiceScore.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/IAsynchronousWriteBufferFactory.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/search/TestAll.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/StressTestConcurrent.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestAll.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestEventReceiver.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestMove.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/ndx/TestAll.java
branches/dev-btm/bigdata/src/test/com/bigdata/sparse/TestAll.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/BigdataZooDefs.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServicesManagerServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/BigdataServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/LoadBalancerConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServicesManagerConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/ZookeeperProcessHelper.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/AdminProxy.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/EmbeddedLoadBalancer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/PrivateInterface.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/ServiceProxy.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/JiniFederation.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/benchmark/ThroughputMaster.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/lookup/AbstractCachingServiceClient.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/lookup/BigdataCachingServiceClient.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/lookup/LoadBalancerClient.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/BroadcastSighup.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/JiniServicesHelper.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/TestAll.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/TestBigdataClient.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/master/TestAll.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/master/TestMappedRDFDataLoadMaster.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/ConcurrentDataLoader.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/RDFLoadTaskFactory.java
branches/dev-btm/src/resources/config/bigdataCluster.config
Added Paths:
-----------
branches/dev-btm/bigdata/src/test/com/bigdata/service/StressTestConcurrentRemote.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestBasicIndexStuffRemote.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestDistributedTransactionServiceRemote.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestEDSRemote.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestEmbeddedClientRemote.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestMetadataIndexRemote.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestMoveRemote.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestOverflowRemote.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestRangeQueryRemote.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestRestartSafeRemote.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestScatterSplitRemote.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestSplitJoinRemote.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/TestBigdataClientRemote.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/master/TestMappedRDFDataLoadMasterRemote.java
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/RuleStats.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/RuleStats.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/RuleStats.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -40,7 +40,6 @@
import com.bigdata.relation.rule.ISlice;
import com.bigdata.relation.rule.IStep;
import com.bigdata.relation.rule.Rule;
-//BTM import com.bigdata.service.ILoadBalancerService;
import com.bigdata.striterator.IKeyOrder;
import com.bigdata.service.LoadBalancer;
@@ -60,8 +59,7 @@
* In order to aggregate the data on rule execution, we want to roll up the data
* for the individual rules along the same lines as the program structure.
*
-BTM * @todo Report as counters aggregated by the {@link ILoadBalancerService}?
-* @todo Report as counters aggregated by the {@link LoadBalancer} service?
+ * @todo Report as counters aggregated by the load balancer service?
*
* @author mikep
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -61,7 +61,6 @@
import com.bigdata.service.EventResource;
import com.bigdata.service.EventType;
import com.bigdata.service.IDataService;
-//BTM import com.bigdata.service.ILoadBalancerService;
import com.bigdata.service.MetadataService;
import com.bigdata.util.InnerCause;
import com.bigdata.util.concurrent.DaemonThreadFactory;
@@ -1327,19 +1326,15 @@
}
/**
-BTM * Return the {@link ILoadBalancerService} if it can be discovered.
-* Return the {@link LoadBalancer} if it can be discovered.
+ * Return the load balancer service if it can be discovered.
*
-BTM * @return the {@link ILoadBalancerService} if it can be discovered and
-* @return the {@link LoadBalancer} service if it can be discovered and
+ * @return the load balancer service if it can be discovered and
* otherwise <code>null</code>.
*/
-//BTM protected ILoadBalancerService getLoadBalancerService() {
-protected LoadBalancer getLoadBalancerService() {
+ protected LoadBalancer getLoadBalancerService() {
// lookup the load balancer service.
-//BTM final ILoadBalancerService loadBalancerService;
-final LoadBalancer loadBalancerService;
+ final LoadBalancer loadBalancerService;
try {
@@ -1377,8 +1372,7 @@
* @param loadBalancerService
* The load balancer.
*/
-//BTM protected boolean shouldMove(final ILoadBalancerService loadBalancerService) {
-protected boolean shouldMove(final LoadBalancer loadBalancerService) {
+ protected boolean shouldMove(final LoadBalancer loadBalancerService) {
if (loadBalancerService == null)
throw new IllegalArgumentException();
@@ -1465,8 +1459,7 @@
* @return The tasks.
*/
private List<AbstractTask> chooseMoves(
-final LoadBalancer loadBalancerService) {
-//BTM final ILoadBalancerService loadBalancerService) {
+ final LoadBalancer loadBalancerService) {
if (resourceManager.maximumMovesPerTarget == 0) {
@@ -2388,8 +2381,7 @@
* Running out of DISK space causes an urgent condition and can lead to
* failure or all services on the same host. Therefore, when a host is near
* to exhausting its DISK space it (a) MUST notify the
-BTM * {@link ILoadBalancerService}; (b) temporary files SHOULD be purged; it
-* {@link ILoadBalancerService}; (b) temporary files SHOULD be purged; it
+ * load balancer service; (b) temporary files SHOULD be purged; it
* MAY choose to shed indices that are "hot for write" since that will slow
* down the rate at which the disk space is consumed; (d) index partitions
* may be aggressively moved off of the LDS; (e) the transaction service MAY
@@ -2464,8 +2456,7 @@
* When it is not available we simply do not consider index
* partition moves.
*/
-//BTM final ILoadBalancerService lbs = getLoadBalancerService();
-final LoadBalancer lbs = getLoadBalancerService();
+ final LoadBalancer lbs = getLoadBalancerService();
if(lbs != null && shouldMove(lbs)) {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/resources/CompactingMergeTask.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/resources/CompactingMergeTask.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/resources/CompactingMergeTask.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -29,7 +29,6 @@
import com.bigdata.service.DataService;
import com.bigdata.service.Event;
import com.bigdata.service.EventResource;
-//BTM import com.bigdata.service.ILoadBalancerService;
import com.bigdata.service.MetadataService;
import com.bigdata.service.ndx.ClientIndexView;
@@ -513,8 +512,7 @@
* critically overloaded, but that should probably be handled by
* different logic.]
*/
-//BTM ILoadBalancerService loadBalancerService = null;
-LoadBalancer loadBalancerService = null;
+ LoadBalancer loadBalancerService = null;
if (vmd.getPercentOfSplit() < resourceManager.maximumMovePercentOfSplit
&& resourceManager.maximumMovesPerTarget != 0
&& resourceManager.getLiveJournal().getName2Addr().rangeCount() > resourceManager.minimumActiveIndexPartitions
@@ -562,19 +560,15 @@
}
/**
-BTM * Return the {@link ILoadBalancerService} if it can be discovered.
-* Return the {@link LoadBalancer} service if it can be discovered.
+ * Return the load balancer service if it can be discovered.
*
-BTM * @return the {@link ILoadBalancerService} if it can be discovered and
-* @return the {@link LoadBalancer} service if it can be discovered and
+ * @return the load balancer service if it can be discovered and
* otherwise <code>null</code>.
*/
-//BTM private ILoadBalancerService getLoadBalancerService() {
-private LoadBalancer getLoadBalancerService() {
+ private LoadBalancer getLoadBalancerService() {
// lookup the load balancer service.
-//BTM final ILoadBalancerService loadBalancerService;
-final LoadBalancer loadBalancerService;
+ final LoadBalancer loadBalancerService;
try {
@@ -612,8 +606,7 @@
* @param loadBalancerService
* The load balancer.
*/
-//BTM protected boolean shouldMove(final ILoadBalancerService loadBalancerService) {
-protected boolean shouldMove(final LoadBalancer loadBalancerService) {
+ protected boolean shouldMove(final LoadBalancer loadBalancerService) {
if (loadBalancerService == null)
throw new IllegalArgumentException();
@@ -692,8 +685,7 @@
* behavior).
*/
private UUID getMoveTarget(final UUID sourceServiceUUID,
-final LoadBalancer loadBalancerService) {
-//BTM final ILoadBalancerService loadBalancerService) {
+ final LoadBalancer loadBalancerService) {
try {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -372,8 +372,7 @@
* Counters that aggregate across all tasks submitted by the client against
* the connected federation. Those counters are sampled by a
* {@link ThreadPoolExecutorStatisticsTask} and reported by the client to
-BTM * the {@link ILoadBalancerService}.
-* the {@link LoadBalancer} service.
+ * the load balancer service.
*/
private final TaskCounters taskCounters = new TaskCounters();
@@ -401,8 +400,7 @@
* performed by the client against the connected federation. These
* {@link TaskCounters} are sampled by a
* {@link ThreadPoolExecutorStatisticsTask} and the sampled data are
-BTM * reported by the client to the {@link ILoadBalancerService}.
-* reported by the client to the {@link LoadBalancer} service.
+ * reported by the client to the load balancer service.
*/
public TaskCounters getTaskCounters() {
@@ -415,8 +413,7 @@
* for this client. There is only a single instance per scale-out index and
* all operations by this client on that index are aggregated by that
* instance. These counters are reported by the client to the
-BTM * {@link ILoadBalancerService}.
-* {@link LoadBalancer} service.
+ * load balancer service.
*
* @param name
* The scale-out index name.
@@ -456,8 +453,7 @@
/**
* Collects interesting statistics on the client's host and process
-BTM * for reporting to the {@link ILoadBalancerService}.
-* for reporting to the {@link LoadBalancer} service.
+ * for reporting to the load balancer service.
*/
private AbstractStatisticsCollector statisticsCollector;
@@ -466,15 +462,13 @@
* or until the federation is {@link #shutdown()}.
* <p>
* Note: Tasks run on this service generally update sampled values on
-BTM * {@link ICounter}s reported to the {@link ILoadBalancerService}. Basic
-* {@link ICounter}s reported to the {@link LoadBalancer} service. Basic
+ * {@link ICounter}s reported to the load balancer service. Basic
* information on the {@link #getExecutorService()} is reported
* automatically. Clients may add additional tasks to report on client-side
* aspects of their application.
* <p>
* Note: Non-sampled counters are automatically conveyed to the
-BTM * {@link ILoadBalancerService} once added to the basic {@link CounterSet}
-* {@link LoadBalancer} service once added to the basic {@link CounterSet}
+ * load balancer service once added to the basic {@link CounterSet}
* returned by {@link #getCounterSet()}.
*
* @param task
@@ -681,8 +675,7 @@
if (dataServiceUUID == null) {
-//BTM final ILoadBalancerService loadBalancerService = getLoadBalancerService();
-final LoadBalancer loadBalancerService = getLoadBalancerService();
+ final LoadBalancer loadBalancerService = getLoadBalancerService();
if (loadBalancerService == null) {
@@ -827,8 +820,7 @@
/**
* Forces the immediate reporting of the {@link CounterSet} to the
-BTM * {@link ILoadBalancerService}. Any errors will be logged, not thrown.
-* {@link LoadBalancer} service. Any errors will be logged, not thrown.
+ * load balancer service. Any errors will be logged, not thrown.
*/
public void reportCounters() {
@@ -999,10 +991,8 @@
* the (required) {@link ReportTask}.
* <p>
* Note: The {@link ReportTask} will relay any collected performance
-BTM * counters to the {@link ILoadBalancerService}, but it also lets the
-BTM * {@link ILoadBalancerService} know which services exist, which is
-* counters to the {@link LoadBalancer} service, but it also lets the
-* {@link LoadBalancer} service know which services exist, which is
+ * counters to the load balancer service, but it also lets the
+ * load balancer service know which services exist, which is
* important for some of its functions.
* <p>
*
@@ -1304,8 +1294,7 @@
/**
* Periodically report performance counter data to the
-BTM * {@link ILoadBalancerService}.
-* {@link LoadBalancer} service.
+ * load balancer service.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
@@ -1382,8 +1371,7 @@
}
-//BTM final ILoadBalancerService loadBalancerService = fed.getLoadBalancerService();
-final LoadBalancer loadBalancerService = fed.getLoadBalancerService();
+ final LoadBalancer loadBalancerService = fed.getLoadBalancerService();
if (loadBalancerService == null) {
System.out.println(">>>>> AbstractFederation.reportPerformanceCounters: loadBalancerService = NULL");
@@ -1483,8 +1471,7 @@
}
/**
-BTM * Queues up an event to be sent to the {@link ILoadBalancerService}.
-* Queues up an event to be sent to the {@link LoadBalancer} service.
+ * Queues up an event to be sent to the load balancer service.
* Events are maintained on a non-blocking queue (no fixed capacity) and
* sent by a scheduled task.
*
@@ -1503,14 +1490,12 @@
}
/**
-BTM * Queue of events sent periodically to the {@link ILoadBalancerService}.
-* Queue of events sent periodically to the {@link LoadBalancer} service.
+ * Queue of events sent periodically to the load balancer service.
*/
final private BlockingQueue<Event> events = new LinkedBlockingQueue<Event>();
/**
-BTM * Sends events to the {@link ILoadBalancerService}.
-* Sends events to the {@link LoadBalancer} service.
+ * Sends events to the load balancer service.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
@@ -1530,8 +1515,7 @@
try {
-//BTM final ILoadBalancerService lbs = getLoadBalancerService();
-final LoadBalancer lbs = getLoadBalancerService();
+ final LoadBalancer lbs = getLoadBalancerService();
if (lbs == null) {
@@ -1583,8 +1567,6 @@
} catch (Throwable t) {
log.warn(getServiceName(), t);
-//BTM
-t.printStackTrace();
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractService.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractService.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractService.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -82,7 +82,7 @@
* <p>
* Several things depend on when this method is invoked, including the setup
* of the per-service {@link CounterSet} reported by the service to the
- * {@link ILoadBalancerService}.
+ * load balancer service.
*
* @param serviceUUID
* The {@link UUID} assigned to the service.
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/DefaultServiceFederationDelegate.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/DefaultServiceFederationDelegate.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/DefaultServiceFederationDelegate.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -46,7 +46,7 @@
/**
* Basic delegate for services that need to override the service UUID and
- * service interface reported to the {@link ILoadBalancerService}.
+ * service interface reported to the load balancer service.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedClient.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedClient.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedClient.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -152,7 +152,8 @@
public static final String DATA_DIR = EmbeddedFederation.class
.getName()
+ ".dataDir";
-
+//BTM
+ public static String SERVICE_IMPL_REMOTE = "serviceImplRemote";
}
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedFederation.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedFederation.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedFederation.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -123,11 +123,13 @@
private final ResourceLockService resourceLockManager;
/**
-BTM * The (in process) {@link LoadBalancerService}.
-* The (in process) {@link LoadBalancer} service.
+ * The (in process) load balancer service.
*/
//BTM private final LoadBalancerService loadBalancerService;
-private final EmbeddedLoadBalancer loadBalancerService;
+//private final EmbeddedLoadBalancer loadBalancerService;
+private final LoadBalancer loadBalancerService;
+private final EmbeddedLoadBalancerServiceImpl remoteLbs;
+private final EmbeddedLoadBalancerImpl lbs;
/**
* The (in process) {@link MetadataService}.
@@ -185,11 +187,9 @@
}
/**
-BTM * The (in process) {@link LoadBalancerService}.
-* The (in process) {@link LoadBalancer} service.
+ * Returns the (in process) load balancer service.
*/
-//BTM final public ILoadBalancerService getLoadBalancerService() {
-final public LoadBalancer getLoadBalancerService() {
+ final public LoadBalancer getLoadBalancerService() {
// Note: return null if service not available/discovered.
@@ -313,6 +313,8 @@
.getProperty(Options.CREATE_TEMP_FILE,
""+Options.DEFAULT_CREATE_TEMP_FILE));
+boolean serviceImplRemote = ( (null != properties.getProperty(EmbeddedClient.Options.SERVICE_IMPL_REMOTE)) ? true : false);
+System.out.println("\n*** serviceImplRemote = "+serviceImplRemote);
/*
* The directory in which the data files will reside.
*/
@@ -617,78 +619,102 @@
if (isTransient) {
-//BTM p.setProperty(LoadBalancerService.Options.TRANSIENT, "true");
-p.setProperty(EmbeddedLoadBalancerServiceImpl.Options.TRANSIENT, "true");
+ p.setProperty(LoadBalancerService.Options.TRANSIENT, "true");
+ p.setProperty(EmbeddedLoadBalancer.Options.TRANSIENT, "true");
-p.setProperty(EmbeddedLoadBalancerServiceImpl.Options.LOG_DIR,
- new File(EmbeddedLoadBalancerServiceImpl.Options.DEFAULT_LOG_DIR).toString());
-
+ p.setProperty(EmbeddedLoadBalancerImpl.Options.LOG_DIR,
+ new File
+ (EmbeddedLoadBalancerImpl.Options.DEFAULT_LOG_DIR).toString());
} else {
-
// specify the data directory for the load balancer.
-
p.setProperty(EmbeddedLoadBalancerServiceImpl.Options.LOG_DIR,
new File(dataDir, "lbs").toString());
-
+ p.setProperty(EmbeddedLoadBalancerImpl.Options.LOG_DIR,
+ new File(dataDir, "lbs").toString());
}
+ if(serviceImplRemote) {
+ try {
+ loadBalancerService =
+ new EmbeddedLoadBalancerServiceImpl
+ (UUID.randomUUID(), p).start();
+ } catch (Throwable t) {
+ log.error(t, t);
+ throw new RuntimeException(t);
+ }
+ remoteLbs = (EmbeddedLoadBalancerServiceImpl)loadBalancerService;
+ lbs = null;
+ } else {
//BTM*** remove after EmbeddedDataServiceImpl/shard.ServiceImpl/EmbeddedDataService
//BTM*** is converted to smart proxy?
-loadBalancerService = new EmbeddedLoadBalancerServiceImpl(UUID.randomUUID(),
- hostname,
- null,//SDM - replace with real SDM after conversion to smart proxy?
+ loadBalancerService =
+ new EmbeddedLoadBalancerImpl
+ (UUID.randomUUID(),
+ hostname,
+ null,//SDM - replace with real SDM after conversion to smart proxy?
//BTM*** EmbeddedDataService.this,
//BTM*** remove after EmbeddedDataService is converted to smart proxy
- dataServiceByUUID,
- p);
+ dataServiceByUUID,
+ p);
+ remoteLbs = null;
+ lbs = (EmbeddedLoadBalancerImpl)loadBalancerService;
+ }
+System.out.println("*** serviceImplRemote = "+serviceImplRemote+" >>> remoteLbs = "+remoteLbs);
+System.out.println("*** serviceImplRemote = "+serviceImplRemote+" >>> lbs = "+lbs);
//BTM*** ------------------------------------------------------------------------------
-
/*
* Have the data services join the load balancer.
*/
for (IDataService ds : this.dataService) {
-
try {
-
- loadBalancerService.join(ds.getServiceUUID(), ds
- .getServiceIface(),
-//BTM
-ds.getServiceName(),
-hostname);
-
+ if(remoteLbs != null) {
+ remoteLbs.join(ds.getServiceUUID(),
+ ds.getServiceIface(),
+ hostname);
+ } else {
+ lbs.join(ds.getServiceUUID(),
+ ds.getServiceIface(),
+ ds.getServiceName(),
+ hostname);
+ }
} catch (IOException e) {
-
// Should never be thrown for an embedded service.
-
log.warn(e.getMessage(), e);
-
}
-
}
/*
* Other service joins.
*/
+ if(remoteLbs != null) {
+ remoteLbs.join
+ (abstractTransactionService.getServiceUUID(),
+ abstractTransactionService.getServiceIface(),
+ hostname);
+ remoteLbs.join(remoteLbs.getServiceUUID(),
+ remoteLbs.getServiceIface(),
+ hostname);
+ remoteLbs.join(metadataService.getServiceUUID(),
+ metadataService.getServiceIface(),
+ hostname);
+ } else {//smart proxy
+
+ lbs.join
+ (abstractTransactionService.getServiceUUID(),
+ abstractTransactionService.getServiceIface(),
+ (abstractTransactionService.getServiceUUID()).toString(),
+ hostname);
+ lbs.join(lbs.getServiceUUID(),
+ lbs.getServiceIface(),
+ (lbs.getServiceUUID()).toString(),
+ hostname);
+ lbs.join(metadataService.getServiceUUID(),
+ metadataService.getServiceIface(),
+ (metadataService.getServiceUUID()).toString(),
+ hostname);
+ }
- loadBalancerService.join(abstractTransactionService.getServiceUUID(),
- abstractTransactionService.getServiceIface(),
-//BTM
-(abstractTransactionService.getServiceUUID()).toString(),
-hostname);
-
- loadBalancerService.join(loadBalancerService.getServiceUUID(),
- loadBalancerService.getServiceIface(),
-//BTM
-(loadBalancerService.getServiceUUID()).toString(),
-hostname);
-
- loadBalancerService.join(metadataService.getServiceUUID(),
- metadataService.getServiceIface(),
-//BTM
-(metadataService.getServiceUUID()).toString(),
-hostname);
-
}
}
@@ -834,41 +860,35 @@
}
-//BTM protected class EmbeddedLoadBalancerServiceImpl extends AbstractEmbeddedLoadBalancerService {
-protected class EmbeddedLoadBalancerServiceImpl extends EmbeddedLoadBalancer {
+ protected class EmbeddedLoadBalancerServiceImpl extends AbstractEmbeddedLoadBalancerService {
/**
* @param serviceUUID
* @param properties
*/
-//BTM public EmbeddedLoadBalancerServiceImpl(UUID serviceUUID, Properties properties) {
-//BTM
-//BTM super(serviceUUID, properties);
-//BTM
-//BTM }
+ public EmbeddedLoadBalancerServiceImpl(UUID serviceUUID, Properties properties) {
+ super(serviceUUID, properties);
+ }
-//BTM @Override
-//BTM public EmbeddedFederation<T> getFederation() {
-//BTM
-//BTM return EmbeddedFederation.this;
-//BTM
-//BTM }
+ @Override
+ public EmbeddedFederation<T> getFederation() {
+ return EmbeddedFederation.this;
+ }
+ }
-public EmbeddedLoadBalancerServiceImpl(UUID serviceUUID,
- String hostname,
- ServiceDiscoveryManager sdm,
-//BTM - remove once EmbeddedDataService converted to smart proxy
-Map<UUID, DataService> dataServiceMap,
- Properties properties)
-{
- super(serviceUUID, hostname,
-sdm,
-properties.getProperty(EmbeddedLoadBalancerServiceImpl.Options.LOG_DIR),
-//BTM*** EmbeddedFederation.this,
-dataServiceMap,//BTM*** - remove after DataService smart proxy?
- properties);
-}
-
+ protected class EmbeddedLoadBalancerImpl extends EmbeddedLoadBalancer {
+
+ public EmbeddedLoadBalancerImpl(UUID serviceUUID,
+ String hostname,
+ ServiceDiscoveryManager sdm,
+ Map<UUID, DataService> dataServiceMap,//BTM - remove once EmbeddedDataService converted to smart proxy?
+ Properties properties)
+ {
+ super(serviceUUID, hostname, sdm,
+ properties.getProperty(EmbeddedLoadBalancerImpl.Options.LOG_DIR),
+ dataServiceMap,//BTM*** - remove after DataService smart proxy?
+ properties);
+ }
}
protected class EmbeddedTransactionServiceImpl extends AbstractEmbeddedTransactionService {
@@ -937,7 +957,11 @@
if (loadBalancerService != null) {
- loadBalancerService.shutdown();
+ if(remoteLbs != null) {
+ remoteLbs.shutdown();
+ } else {
+ lbs.shutdown();
+ }
// loadBalancerService = null;
@@ -981,7 +1005,11 @@
if (loadBalancerService != null) {
- loadBalancerService.shutdownNow();
+ if(remoteLbs != null) {
+ remoteLbs.shutdownNow();
+ } else {
+ lbs.shutdownNow();
+ }
// loadBalancerService = null;
@@ -1026,9 +1054,12 @@
metadataService.destroy();
}
+ if(remoteLbs != null) {
+ remoteLbs.destroy();
+ } else {
+ lbs.destroy();
+ }
- loadBalancerService.destroy();
-
if (!isTransient && !dataDir.delete()) {
log.warn(ERR_COULD_NOT_DELETE + dataDir);
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/Event.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/Event.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/Event.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -48,10 +48,8 @@
/**
* An event. Events are queued by the {@link IBigdataClient} and self-reported
-BTM * periodically to the {@link ILoadBalancerService}. The event is assigned a
-BTM * {@link UUID} when it is created and the {@link ILoadBalancerService} assigned
-* periodically to the {@link LoadBalancer} service. The event is assigned a
-* {@link UUID} when it is created and the {@link LoadBalancer} service assigned
+ * periodically to the load balancer service. The event is assigned a
+ * {@link UUID} when it is created and the load balancer assigned
* start and end event times based on its local clock as the events are received
* (this helps to reduce the demand on the {@link ITransactionService} for
* global timestamp).
@@ -60,8 +58,7 @@
* @version $Id$
*
* @todo compact event serialization when reporting to the
-BTM * {@link ILoadBalancerService}, including factoring out of the common
-* {@link LoadBalancer} service, including factoring out of the common
+ * load balancer service, including factoring out of the common
* metadata (some stuff will always be the same for a given reported and
* does not need to be reported with each event).
*
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/HostScore.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/HostScore.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/HostScore.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -1,20 +1,13 @@
package com.bigdata.service;
-//BTM import com.bigdata.service.LoadBalancerService.UpdateTask;
-
/**
-BTM * Per-host metadata and a score for that host which gets updated
-BTM * periodically by {@link UpdateTask}. {@link HostScore}s are a
-BTM * <em>resource utilization</em> measure. They are higher for a host which
-BTM * is more highly utilized. There are several ways to look at the score,
-BTM * including the {@link #rawScore}, the {@link #rank}, and the
-BTM * {@link #drank normalized double-precision rank}. The ranks move in the
- * Per-host metadata and a score for that host which gets updated periodically
- * by <code>com.bigdata.loadbalancer.EmbeddedLoadBalancer.UpdateTask</code>.
- * {@link HostScore}s are a <em>resource utilization</em> measure. They are
- * higher for a host which is more highly utilized. There are several ways to
- * look at the score, including the {@link #rawScore}, the {@link #rank}, and
- * the {@link #drank normalized double-precision rank}. The ranks move in the
+ * Per-host metadata and a score for that host which gets updated
+ * periodically by the <code>UpdateTask</code> of the current
+ * implementation of the load balancer service. {@link HostScore}s are a
+ * <em>resource utilization</em> measure. They are higher for a host which
+ * is more highly utilized. There are several ways to look at the score,
+ * including the {@link #rawScore}, the {@link #rank}, and the
+ * {@link #drank normalized double-precision rank}. The ranks move in the
* same direction as the {@link #rawScore}s - a higher rank indicates
* higher utilization. The least utilized host is always rank zero (0). The
* most utilized host is always in the last rank.
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/IBigdataClient.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/IBigdataClient.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/IBigdataClient.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -447,9 +447,8 @@
String DEFAULT_COLLECT_QUEUE_STATISTICS = "true";
/**
- * The delay between reports of performance counters to the
-BTM * {@link ILoadBalancerService} in milliseconds ({@value #DEFAULT_REPORT_DELAY}).
-* {@link LoadBalancer} service in milliseconds ({@value #DEFAULT_REPORT_DELAY}).
+ * The delay between reports of performance counters to the load
+ * balancer service in milliseconds ({@value #DEFAULT_REPORT_DELAY}).
*
* @see #DEFAULT_REPORT_DELAY
*/
@@ -467,12 +466,10 @@
* used. The httpd service may be disabled by specifying <code>-1</code>
* as the port.
* <p>
-BTM * Note: The httpd service for the {@link LoadBalancerService} is
-* Note: The httpd service for the {@link LoadBalancer} service is
+ * Note: The httpd service for the load balancer service is
* normally run on a known port in order to make it easy to locate that
* service, e.g., port 80, 8000 or 8080, etc. This MUST be overridden for
-BTM * the {@link LoadBalancerService} it its configuration since
-* the {@link LoadBalancer} service in its configuration since
+ * the load balancer service in its configuration since
* {@link #DEFAULT_HTTPD_PORT} will otherwise cause a random port to be
* assigned.
*/
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/IBigdataFederation.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/IBigdataFederation.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/IBigdataFederation.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -84,8 +84,7 @@
*
* @return The service -or- <code>null</code> if the service has not been discovered.
*/
-//BTM public ILoadBalancerService getLoadBalancerService();
-public LoadBalancer getLoadBalancerService();
+ public LoadBalancer getLoadBalancerService();
/**
* Return the metadata service (or a proxy for the metadata service).
@@ -103,13 +102,11 @@
/**
* The {@link CounterSet} which the client will use report its statistics to
-BTM * the {@link ILoadBalancerService}.
-* the {@link LoadBalancer} service.
+ * the load balancer service.
* <p>
* Note: Applications MAY add their own counters (within a suitable
* namespace) to the returned {@link CounterSet} in order to report their
-BTM * own performance data to the {@link ILoadBalancerService}.
-* own performance data to the {@link LoadBalancer} service.
+ * own performance data to the load balancer service.
*
* @see #getServiceCounterSet()
* @see #getServiceCounterPathPrefix()
@@ -183,8 +180,7 @@
* and which is part of the connected federation.
* <p>
* Note: This method is here as a failsafe when the
-BTM * {@link ILoadBalancerService} is not available.
-* {@link LoadBalancer} service is not available.
+ * load balancer service is not available.
*
* @return <code>null</code> if there are NO known {@link IDataService}s.
*/
@@ -251,8 +247,7 @@
* {@link IndexMetadata.Options#INITIAL_DATA_SERVICE} was
* specified, then the identified service will be used. Otherwise
* an underutilized service will be selected using the
-BTM * {@link ILoadBalancerService}.
-* {@link LoadBalancer} service.
+ * load balancer service.
*
* @return The UUID of the registered index.
*
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/IFederationDelegate.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/IFederationDelegate.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/IFederationDelegate.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -61,8 +61,7 @@
/**
* Return the class or interface that is the most interesting facet of the
* client and which will be used to identify this client in the performance
-BTM * counters reported to the {@link ILoadBalancerService}.
-* counters reported to the {@link LoadBalancer} service.
+ * counters reported to the load balancer service.
*
* @return The class or interface and never <code>null</code>.
*/
@@ -80,8 +79,7 @@
* Offers the service an opportunity to dynamically detach and re-attach
* performance counters. This can be invoked either in response to an http
* GET or the periodic reporting of performance counters to the
-BTM * {@link ILoadBalancerService}. In general, implementations should limit
-* {@link LoadBalancer} service. In general, implementations should limit
+ * load balancer service. In general, implementations should limit
* the frequency of update, e.g., to no more than once a second.
*/
public void reattachDynamicCounters();
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/ILoadBalancerService.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/ILoadBalancerService.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/ILoadBalancerService.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -28,179 +28,18 @@
package com.bigdata.service;
-import java.io.IOException;
-import java.util.UUID;
-import java.util.concurrent.TimeoutException;
-
/**
- * Interface for collecting, reporting, and decision-making based on node and
- * service utilization statistics.
+ * <code>Remote</code> interface for collecting, reporting, and
+ * decision-making based on node and service utilization statistics.
+ *
+ * @see LoadBalancer
+ * @see IService
+ * @see IEventReceivingService
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
-public interface ILoadBalancerService extends IService, IEventReceivingService {
+public interface ILoadBalancerService
+ extends LoadBalancer, IService, IEventReceivingService
+{
- /**
- * Send performance counters. Clients SHOULD invoke this method no less than
- * once every 60 seconds.
- *
- * @param serviceUUID
- * The service {@link UUID} that is self-reporting.
- * @param data
- * The serialized performance counter data.
- *
- * @throws IOException
- */
- public void notify(UUID serviceUUID, byte[] data) throws IOException;
-
- /**
- * A warning issued by a client when it is in danger of depleting its
- * resources.
- *
- * @param msg
- * A message.
- * @param serviceUUID
- * The service {@link UUID} that is self-reporting.
- *
- * @throws IOException
- */
- public void warn(String msg,UUID serviceUUID) throws IOException;
-
- /**
- * An urgent warning issued the caller is in immediate danger of depleting
- * its resources with a consequence of immediate service and/or host
- * failure(s).
- *
- * @param msg
- * A message.
- * @param serviceUUID
- * The service {@link UUID} that is self-reporting.
- *
- * @throws IOException
- */
- public void urgent(String msg,UUID serviceUUID) throws IOException;
-
- /**
- * Return the {@link UUID} of an under-utilized data service. If there is no
- * under-utilized service, then return the {@link UUID} of the service with
- * the least load.
- *
- * @throws TimeoutException
- * if there are no data services and a timeout occurs while
- * awaiting a service join.
- *
- * @throws InterruptedException
- * if the request is interrupted.
- */
- public UUID getUnderUtilizedDataService() throws IOException, TimeoutException, InterruptedException;
-
- /**
- * Return up to <i>limit</i> {@link IDataService} {@link UUID}s that are
- * currently under-utilized.
- * <p>
- * When <i>minCount</i> is positive, this method will always return at
- * least <i>minCount</i> service {@link UUID}s, however the {@link UUID}s
- * returned MAY contain duplicates if the {@link LoadBalancerService} has a
- * strong preference for allocating load to some services (or for NOT
- * allocating load to other services). Further, the
- * {@link LoadBalancerService} MAY choose (or be forced to choose) to return
- * {@link UUID}s for services that are within a nominal utilization range,
- * or even {@link UUID}s for services that are highly-utilized if it could
- * otherwise not satisify the request.
- *
- * @param minCount
- * The minimum #of services {@link UUID}s to return -or- zero
- * (0) if there is no minimum limit.
- * @param maxCount
- * The maximum #of services {@link UUID}s to return -or- zero
- * (0) if there is no maximum limit.
- * @param exclude
- * The optional {@link UUID} of a data service to be excluded
- * from the returned set.
- *
- * @return Up to <i>maxCount</i> under-utilized services -or-
- * <code>null</code> IFF no services are recommended at this time
- * as needing additional load.
- *
- * @throws TimeoutException
- * if there are no data services, or if there is only a single
- * data service and it is excluded by the request, and a timeout
- * occurs while awaiting a service join.
- *
- * @throws InterruptedException
- * if the request is interrupted.
- *
- * @todo generalize to also accept the class or interface of the service so
- * that it can be used with services other than data services, e.g.,
- * metadata services, map/reduce services, {@link IBigdataClient}s,
- * etc.
- *
- * @todo probably should use {@link Integer#MAX_VALUE} rather than ZERO for
- * the "no limit" signifier for [maxCount].
- */
- public UUID[] getUnderUtilizedDataServices(int minCount, int maxCount,
- UUID exclude) throws IOException, TimeoutException, InterruptedException;
-
- /**
- * Return <code>true</code> if the service is considered to be "highly
- * utilized".
- * <p>
- * Note: This is used mainly to decide when a service should attempt to shed
- * index partitions. This implementation SHOULD reflect the relative rank of
- * the service among all services as well as its absolute load.
- *
- * @param serviceUUID
- * The service {@link UUID}.
- *
- * @return <code>true</code> if the service is considered to be "highly
- * utilized".
- *
- * @throws IOException
- */
- public boolean isHighlyUtilizedDataService(UUID serviceUUID) throws IOException;
-
- /**
- * Return <code>true</code> if the service is considered to be
- * "under-utilized".
- *
- * @param serviceUUID
- * The service {@link UUID}.
- *
- * @return <code>true</code> if the service is considered to be "under-utilized".
- *
- * @throws IOException
- */
- public boolean isUnderUtilizedDataService(UUID serviceUUID) throws IOException;
-
- /**
- * Logs counters to a temp file. Replacement for sighup mechanism.
- */
- public void sighup() throws IOException;
-
-// /**
-// * Return the identifier(s) of under-utilized service(s).
-// *
-// * @param minCount
-// * The minimum #of services {@link UUID}s to return -or- zero
-// * (0) if there is no minimum limit.
-// * @param maxCount
-// * The maximum #of services {@link UUID}s to return -or- zero
-// * (0) if there is no maximum limit.
-// * @param exclude
-// * The optional {@link UUID} of a service to be excluded from the
-// * returned set.
-// * @param iface
-// * A class or interface that the service must implement.
-// *
-// * @return Up to <i>limit</i> under-utilized services -or-
-// * <code>null</code> IFF no services are recommended at this time
-// * as needing additional load.
-// *
-// * @todo Since {@link IMetadataService} extends {@link IDataService} we
-// * really need a filter here that can detect the difference.
-// */
-// public UUID[] getUnderUtilizedService(int minCount, int maxCount, UUID exclude,Class iface)
-// throws IOException;
-
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/IServiceLoadHelper.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/IServiceLoadHelper.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/IServiceLoadHelper.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -23,9 +23,8 @@
* @throws InterruptedException
* @throws TimeoutException
*
-BTM * @see ILoadBalancerService#getUnderUtilizedDataServices(int, int,
-* @see LoadBalancer#getUnderUtilizedDataServices(int, int,
- * UUID)
+ * @see getUnderUtilizedDataServices(int, int, UUID) of the load balancer
+ * service interface
*/
public UUID[] getUnderUtilizedDataServices(int minCount, int maxCount,
UUID exclude) throws InterruptedException, TimeoutException;
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/LoadBalancer.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/LoadBalancer.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/LoadBalancer.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -32,7 +32,7 @@
* Interface for collecting, reporting, and decision-making based on node and
* service utilization statistics.
*/
-public interface LoadBalancer extends Service {
+public interface LoadBalancer {
/**
* This method is called by an entity that wishes to send its
@@ -175,4 +175,16 @@
* that calls this method.
*/
boolean isUnderUtilizedDataService(UUID serviceId) throws IOException;
+
+
+ /**
+ * Special method that instructs the load balancer service to log the
+ * counters managed by that service to a temporary file. This method
+ * is intended as replacement for the system based sighup mechanism.
+ *
+ * @throws IOException if there is a communication failure between
+ * this load balancer service and the entity
+ * that calls this method.
+ */
+ public void sighup() throws IOException;
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/ServiceScore.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/ServiceScore.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/ServiceScore.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -2,28 +2,17 @@
import java.util.UUID;
-//BTM import com.bigdata.service.LoadBalancerService.UpdateTask;
-
/**
-BTM * Per-service metadata and a score for that service which gets updated
-BTM * periodically by the {@link UpdateTask}. {@link ServiceScore}s are a
-BTM * <em>resource utilization</em> measure. They are higher for a service
-BTM * which is more highly utilized. There are several ways to look at the
-BTM * score, including the {@link #rawScore}, the {@link #rank}, and the
-BTM * {@link #drank normalized double-precision rank}. The ranks move in the
-BTM * same direction as the {@link #rawScore}s - a higher rank indicates
-BTMBTM * higher utilization. The least utilized service is always rank zero (0).
-BTM * The most utilized service is always in the last rank.
- * Per-service metadata and a score for that service which gets
- * updated periodically by
- * <code>com.bigdata.loadbalancer.EmbeddedLoadBalancer.UpdateTask</code>.
- * {@link ServiceScore}s are a <em>resource utilization</em> measure. They
- * are higher for a service which is more highly utilized. There are
- * several ways to look at the score, including the {@link #rawScore}, the
- * {@link #rank}, and the {@link #drank normalized double-precision rank}.
- * The ranks move in the same direction as the {@link #rawScore}s - a higher
- * rank indicates higher utilization. The least utilized service is always
- * rank zero (0). The most utilized service is always in the last rank.
+ * Per-service metadata and a score for that service which gets updated
+ * periodically by the <code>UpdateTask</code> of the current
+ * implementation of the load balancer service. {@link ServiceScore}s are a
+ * <em>resource utilization</em> measure. They are higher for a service
+ * which is more highly utilized. There are several ways to look at the
+ * score, including the {@link #rawScore}, the {@link #rank}, and the
+ * {@link #drank normalized double-precision rank}. The ranks move in the
+ * same direction as the {@link #rawScore}s - a higher rank indicates
+ * higher utilization. The least utilized service is always rank zero (0).
+ * The most utilized service is always in the last rank.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/IAsynchronousWriteBufferFactory.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/IAsynchronousWriteBufferFactory.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/IAsynchronousWriteBufferFactory.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -41,7 +41,6 @@
import com.bigdata.relation.accesspath.IRunnableBuffer;
import com.bigdata.service.AbstractFederation;
import com.bigdata.service.IBigdataFederation;
-//BTM import com.bigdata.service.ILoadBalancerService;
import com.bigdata.service.Split;
import com.bigdata.service.ndx.pipeline.IDuplicateRemover;
import com.bigdata.service.ndx.pipeline.IndexAsyncWriteStats;
@@ -87,8 +86,7 @@
* {@link Future} will not terminate (other than by error) until the buffer
* has been {@link IBlockingBuffer#close() closed}. The {@link Future}
* evaluates to an {@link IndexAsyncWriteStats} object. Those statistics are
-BTM * also reported to the {@link ILoadBalancerService} via the
-* also reported to the {@link LoadBalancer} service via the
+ * also reported to the load balancer service via the
* {@link IBigdataFederation}.
* <p>
* Each buffer returned by this method is independent, and writes onto
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerTestCase.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerTestCase.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerTestCase.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -69,7 +69,6 @@
import com.bigdata.service.IBigdataClient;
import com.bigdata.service.IBigdataFederation;
import com.bigdata.service.IDataService;
-//BTM import com.bigdata.service.ILoadBalancerService;
import com.bigdata.service.IMetadataService;
import com.bigdata.service.IService;
import com.bigdata.service.Session;
@@ -479,8 +478,7 @@
return null;
}
-//BTM public ILoadBalancerService getLoadBalancerService() {
-public LoadBalancer getLoadBalancerService() {
+ public LoadBalancer getLoadBalancerService() {
return null;
}
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/search/TestAll.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/search/TestAll.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/search/TestAll.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -35,6 +35,7 @@
import com.bigdata.journal.AbstractIndexManagerTestCase;
import com.bigdata.journal.IIndexManager;
import com.bigdata.service.TestEDS;
+import com.bigdata.service.TestEDSRemote;
import com.bigdata.service.TestJournal;
/**
@@ -75,6 +76,7 @@
// search backed by EDS.
suite.addTest(proxySuite(new TestEDS("EDS Search"),"EDS"));
+ suite.addTest(proxySuite(new TestEDSRemote("EDS Search Remote"),"EDS Remote"));
/* For EDS:
*
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/service/StressTestConcurrent.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/service/StressTestConcurrent.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/service/StressTestConcurrent.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -171,15 +171,15 @@
* Note: Disables the initial round robin policy for the load balancer
* service so that it will use our fakes scores.
*/
-//BTM properties.setProperty(
-//BTM LoadBalancerService.Options.INITIAL_ROUND_ROBIN_UPDATE_COUNT,
-//BTM "0");
+ properties.setProperty(
+ LoadBalancerService.Options.INITIAL_ROUND_ROBIN_UPDATE_COUNT,
+ "0");
properties.setProperty(EmbeddedLoadBalancer.Options.INITIAL_ROUND_ROBIN_UPDATE_COUNT, "0");
// load balancer update delay
// properties.setProperty(LoadBalancerService.Options.UPDATE_DELAY,"10000");
/...
[truncated message content] |
|
From: <btm...@us...> - 2010-08-08 15:57:50
|
Revision: 3433
http://bigdata.svn.sourceforge.net/bigdata/?rev=3433&view=rev
Author: btmurphy
Date: 2010-08-08 15:57:40 +0000 (Sun, 08 Aug 2010)
Log Message:
-----------
merge -r3378:HEAD(3430) ~/bigdata/trunk ~/bigdata/branches/dev-btm [trunk --> branch dev-btm]
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/BTree.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/DumpIndexSegment.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexMetadata.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java
branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java
branches/dev-btm/bigdata/src/java/com/bigdata/io/DirectBufferPool.java
branches/dev-btm/bigdata/src/java/com/bigdata/io/WriteCache.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractJournal.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java
branches/dev-btm/bigdata/src/java/com/bigdata/mdi/PartitionLocator.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/AbstractResource.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/IMutableResource.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/RelationFusedView.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/locator/DefaultResourceLocator.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/locator/ILocatableResource.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/BTreeMetadata.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/IndexManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/OverflowManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/ResourceEvents.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/StoreManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractTransactionService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/DataService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/DefaultServiceFederationDelegate.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/DistributedTransactionService.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/GlobalRowStoreHelper.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/TPS.java
branches/dev-btm/bigdata/src/java/com/bigdata/striterator/ChunkedConvertingIterator.java
branches/dev-btm/bigdata/src/resources/logging/log4j.properties
branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractIndexSegmentTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexSegmentMultiBlockIterators.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java
branches/dev-btm/bigdata/src/test/com/bigdata/concurrent/StressTestNonBlockingLockManagerWithTxDag.java
branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestAll.java
branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestTransactionService.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerBootstrapTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestReleaseResources.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/StressTestConcurrent.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestDistributedTransactionServiceRestart.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/ndx/pipeline/TestMasterTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/TransactionServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/lookup/AbstractCachingServiceClient.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/TaskMaster.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/DumpFederation.java
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query9.txt
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/DefaultExtensionFactory.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtension.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtensionFactory.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/ILexiconConfiguration.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDDecimalIV.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdWriteProc.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdWriteTask.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFDataLoadMaster.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/AbstractRuleFastClosure_3_5_6_7_9.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/RDFJoinNexus.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/LocalTripleStore.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util/Splitter.config
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util/Splitter.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/internal/ColorsEnumExtension.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/internal/EpochExtension.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/internal/SampleExtensionFactory.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestEncodeDecodeKeys.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/AbstractRIOTestCase.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/TestAsynchronousStatementBufferFactory.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/small.rdf
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/AbstractTestCase.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/TestAll.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/TestScaleOutTripleStoreWithEmbeddedFederation.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/TestScaleOutTripleStoreWithJiniFederation.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl2.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuadsAndPipelineJoins.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuadsAndPipelineJoinsWithoutInlining.java
branches/dev-btm/build.xml
branches/dev-btm/src/resources/analysis/queries/benchmark.txt
branches/dev-btm/src/resources/config/README
branches/dev-btm/src/resources/config/bigdataCluster.config
branches/dev-btm/src/resources/config/bigdataCluster16.config
branches/dev-btm/src/resources/config/log4j.properties
branches/dev-btm/src/resources/scripts/bigdata.initd
Added Paths:
-----------
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query9-modified.txt
branches/dev-btm/src/resources/config/bigdataStandalone.config
branches/dev-btm/src/resources/scripts/dumpFed.sh
branches/dev-btm/src/resources/scripts/nanoSparqlServer.sh
Property Changed:
----------------
branches/dev-btm/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/attr/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/disco/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/
branches/dev-btm/bigdata-perf/
branches/dev-btm/bigdata-perf/lubm/lib/
branches/dev-btm/bigdata-perf/lubm/src/resources/
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util/
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/
branches/dev-btm/dsi-utils/LEGAL/
branches/dev-btm/dsi-utils/lib/
branches/dev-btm/dsi-utils/src/
branches/dev-btm/dsi-utils/src/test/
branches/dev-btm/dsi-utils/src/test/it/
branches/dev-btm/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/osgi/
branches/dev-btm/src/resources/config/
Property changes on: branches/dev-btm
___________________________________________________________________
Modified: svn:ignore
- ant-build
src
bin
bigdata*.jar
ant-release
standalone
test*
countersfinal.xml
events.jnl
.settings
*.jnl
TestInsertRate.out
SYSTAP-BBT-result.txt
U10load+query
*.hprof
com.bigdata.cache.TestHardReferenceQueueWithBatchingUpdates.exp.csv
commit-log.txt
eventLog
dist
bigdata-test
com.bigdata.rdf.stress.LoadClosureAndQueryTest.*.csv
+ ant-build
src
bin
bigdata*.jar
ant-release
standalone
test*
countersfinal.xml
events.jnl
.settings
*.jnl
TestInsertRate.out
SYSTAP-BBT-result.txt
U10load+query
*.hprof
com.bigdata.cache.TestHardReferenceQueueWithBatchingUpdates.exp.csv
commit-log.txt
eventLog
dist
bigdata-test
com.bigdata.rdf.stress.LoadClosureAndQueryTest.*.csv
DIST.*.tgz
REL.*.tgz
Modified: svn:mergeinfo
- /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/LEXICON_REFACTOR_BRANCH:2633-3304
/branches/bugfix-btm:2594-3237
/branches/fko:3150-3194
/trunk:2575-2594,2596-2877,2882-2903,2910-3378
+ /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/LEXICON_REFACTOR_BRANCH:2633-3304
/branches/bugfix-btm:2594-3237
/branches/fko:3150-3194
/trunk:2575-2594,2596-2877,2882-2903,2910-3430
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -420,7 +420,7 @@
}
}
-
+
/**
* Note: A commit is required in order for a read-committed view to have
* access to the registered indices. When running against an
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -2840,7 +2840,8 @@
* might also want to limit the maximum size of the reads.
*/
- final DirectBufferPool pool = DirectBufferPool.INSTANCE_10M;
+// final DirectBufferPool pool = DirectBufferPool.INSTANCE_10M;
+ final DirectBufferPool pool = DirectBufferPool.INSTANCE;
if (true
&& ((flags & REVERSE) == 0)
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/BTree.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/BTree.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/BTree.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -644,7 +644,18 @@
this.lastCommitTime = lastCommitTime;
}
- private long lastCommitTime = 0L;// Until the first commit.
+
+ /**
+ * The lastCommitTime of the {@link Checkpoint} record from which the
+ * {@link BTree} was loaded.
+ * <p>
+ * Note: Made volatile on 8/2/2010 since it is not otherwise obvious what
+ * would guarantee visibility of this field, through I do seem to remember
+ * that visibility might be guaranteed by how the BTree class is discovered
+ * and returned to the class. Still, it does no harm to make this a volatile
+ * read.
+ */
+ volatile private long lastCommitTime = 0L;// Until the first commit.
/**
* Return the {@link IDirtyListener}.
@@ -1525,45 +1536,63 @@
}
- /**
- * Load an instance of a {@link BTree} or derived class from the store. The
- * {@link BTree} or derived class MUST declare a constructor with the
- * following signature: <code>
+ /**
+ * Load an instance of a {@link BTree} or derived class from the store. The
+ * {@link BTree} or derived class MUST declare a constructor with the
+ * following signature: <code>
*
* <i>className</i>(IRawStore store, Checkpoint checkpoint, BTreeMetadata metadata, boolean readOnly)
*
* </code>
- *
- * @param store
- * The store.
- * @param addrCheckpoint
- * The address of a {@link Checkpoint} record for the index.
- * @param readOnly
- * When <code>true</code> the {@link BTree} will be marked as
- * read-only. Marking has some advantages relating to the locking
- * scheme used by {@link Node#getChild(int)} since the root node
- * is known to be read-only at the time that it is allocated as
- * per-child locking is therefore in place for all nodes in the
- * read-only {@link BTree}. It also results in much higher
- * concurrency for {@link AbstractBTree#touch(AbstractNode)}.
- *
- * @return The {@link BTree} or derived class loaded from that
- * {@link Checkpoint} record.
- */
+ *
+ * @param store
+ * The store.
+ * @param addrCheckpoint
+ * The address of a {@link Checkpoint} record for the index.
+ * @param readOnly
+ * When <code>true</code> the {@link BTree} will be marked as
+ * read-only. Marking has some advantages relating to the locking
+ * scheme used by {@link Node#getChild(int)} since the root node
+ * is known to be read-only at the time that it is allocated as
+ * per-child locking is therefore in place for all nodes in the
+ * read-only {@link BTree}. It also results in much higher
+ * concurrency for {@link AbstractBTree#touch(AbstractNode)}.
+ *
+ * @return The {@link BTree} or derived class loaded from that
+ * {@link Checkpoint} record.
+ *
+ * @throws IllegalArgumentException
+ * if store is <code>null</code>.
+ */
@SuppressWarnings("unchecked")
public static BTree load(final IRawStore store, final long addrCheckpoint,
final boolean readOnly) {
+ if (store == null)
+ throw new IllegalArgumentException();
+
/*
* Read checkpoint record from store.
*/
- final Checkpoint checkpoint = Checkpoint.load(store, addrCheckpoint);
+ final Checkpoint checkpoint;
+ try {
+ checkpoint = Checkpoint.load(store, addrCheckpoint);
+ } catch (Throwable t) {
+ throw new RuntimeException("Could not load Checkpoint: store="
+ + store + ", addrCheckpoint="
+ + store.toString(addrCheckpoint), t);
+ }
- /*
- * Read metadata record from store.
- */
- final IndexMetadata metadata = IndexMetadata.read(store, checkpoint
- .getMetadataAddr());
+ /*
+ * Read metadata record from store.
+ */
+ final IndexMetadata metadata;
+ try {
+ metadata = IndexMetadata.read(store, checkpoint.getMetadataAddr());
+ } catch (Throwable t) {
+ throw new RuntimeException("Could not read IndexMetadata: store="
+ + store + ", checkpoint=" + checkpoint, t);
+ }
if (log.isInfoEnabled()) {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/DumpIndexSegment.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/DumpIndexSegment.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/DumpIndexSegment.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -36,6 +36,7 @@
import org.apache.log4j.Logger;
import com.bigdata.btree.IndexSegment.ImmutableNodeFactory.ImmutableLeaf;
+import com.bigdata.io.DirectBufferPool;
import com.bigdata.journal.DumpJournal;
import com.bigdata.rawstore.IRawStore;
@@ -154,6 +155,16 @@
}
+ // multi-block scan of the index segment.
+ boolean multiBlockScan = false; // @todo command line option.
+ if (multiBlockScan) {
+
+ writeBanner("dump leaves using multi-block forward scan");
+
+ dumpLeavesMultiBlockForwardScan(store);
+
+ }
+
// dump the leaves using a fast reverse scan.
boolean fastReverseScan = true;// @todo command line option
if (fastReverseScan) {
@@ -524,6 +535,36 @@
}
+ /**
+ * Dump leaves using the {@link IndexSegmentMultiBlockIterator}.
+ *
+ * @param store
+ */
+ static void dumpLeavesMultiBlockForwardScan(final IndexSegmentStore store) {
+
+ final long begin = System.currentTimeMillis();
+
+ final IndexSegment seg = store.loadIndexSegment();
+
+ final ITupleIterator<?> itr = new IndexSegmentMultiBlockIterator(seg, DirectBufferPool.INSTANCE,
+ null/* fromKey */, null/* toKey */, IRangeQuery.DEFAULT/* flags */);
+
+ int nscanned = 0;
+
+ while(itr.hasNext()) {
+
+ itr.next();
+
+ nscanned++;
+
+ }
+
+ final long elapsed = System.currentTimeMillis() - begin;
+
+ System.out.println("Visited "+nscanned+" tuples using multi-block forward scan in "+elapsed+" ms");
+
+ }
+
static void writeBanner(String s) {
System.out.println(bar);
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexMetadata.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexMetadata.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexMetadata.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -2049,10 +2049,14 @@
// Note: default assumes NOT an index partition.
this.pmd = null;
+ /* Intern'd to reduce duplication on the heap. Will be com.bigdata.btree.BTree or
+ * com.bigdata.btree.IndexSegment and occasionally a class derived from BTree.
+ */
this.btreeClassName = getProperty(indexManager, properties, namespace,
- Options.BTREE_CLASS_NAME, BTree.class.getName().toString());
+ Options.BTREE_CLASS_NAME, BTree.class.getName()).intern();
- this.checkpointClassName = Checkpoint.class.getName();
+ // Intern'd to reduce duplication on the heap.
+ this.checkpointClassName = Checkpoint.class.getName().intern();
// this.addrSer = AddressSerializer.INSTANCE;
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -207,7 +207,7 @@
if (properties != null) {
- val = properties.getProperty(key, def);
+ val = properties.getProperty(key);//, def);
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -108,7 +108,7 @@
}
- ICUSortKeyGenerator(Locale locale, Object strength, DecompositionEnum mode) {
+ ICUSortKeyGenerator(final Locale locale, final Object strength, final DecompositionEnum mode) {
if (locale == null)
throw new IllegalArgumentException();
@@ -132,7 +132,7 @@
} else {
- StrengthEnum str = (StrengthEnum) strength;
+ final StrengthEnum str = (StrengthEnum) strength;
if (log.isInfoEnabled())
log.info("strength=" + str);
@@ -200,9 +200,9 @@
* Buffer is reused for each {@link String} from which a sort key is
* derived.
*/
- private RawCollationKey raw = new RawCollationKey(128);
+ final private RawCollationKey raw = new RawCollationKey(128);
- public void appendSortKey(KeyBuilder keyBuilder, String s) {
+ public void appendSortKey(final KeyBuilder keyBuilder, final String s) {
// RawCollationKey raw = collator.getRawCollationKey(s, null);
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -278,19 +278,19 @@
AbstractStatisticsCollector
.addGarbageCollectorMXBeanCounters(serviceRoot
.makePath(ICounterHierarchy.Memory_GarbageCollectors));
-
- /*
- * Add counters reporting on the various DirectBufferPools.
- */
- {
- // general purpose pool.
- serviceRoot.makePath(
- IProcessCounters.Memory + ICounterSet.pathSeparator
- + "DirectBufferPool").attach(
- DirectBufferPool.getCounters());
-
- }
+ // Moved since counters must be dynamically reattached to reflect pool hierarchy.
+// /*
+// * Add counters reporting on the various DirectBufferPools.
+// */
+// {
+//
+// serviceRoot.makePath(
+// IProcessCounters.Memory + ICounterSet.pathSeparator
+// + "DirectBufferPool").attach(
+// DirectBufferPool.getCounters());
+//
+// }
if (LRUNexus.INSTANCE != null) {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/io/DirectBufferPool.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/io/DirectBufferPool.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/io/DirectBufferPool.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -218,12 +218,12 @@
*/
public final static DirectBufferPool INSTANCE;
- /**
- * A JVM-wide pool of direct {@link ByteBuffer}s with a default
- * {@link Options#BUFFER_CAPACITY} of <code>10 MB</code>. The main use case
- * for the 10M buffers are multi-block IOs for the {@link IndexSegment}s.
- */
- public final static DirectBufferPool INSTANCE_10M;
+// /**
+// * A JVM-wide pool of direct {@link ByteBuffer}s with a default
+// * {@link Options#BUFFER_CAPACITY} of <code>10 MB</code>. The main use case
+// * for the 10M buffers are multi-block IOs for the {@link IndexSegment}s.
+// */
+// public final static DirectBufferPool INSTANCE_10M;
/**
* An unbounded list of all {@link DirectBufferPool} instances.
@@ -251,11 +251,11 @@
bufferCapacity//
);
- INSTANCE_10M = new DirectBufferPool(//
- "10M",//
- Integer.MAX_VALUE, // poolCapacity
- 10 * Bytes.megabyte32 // bufferCapacity
- );
+// INSTANCE_10M = new DirectBufferPool(//
+// "10M",//
+// Integer.MAX_VALUE, // poolCapacity
+// 10 * Bytes.megabyte32 // bufferCapacity
+// );
/*
* This configuration will block if there is a concurrent demand for
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/io/WriteCache.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/io/WriteCache.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/io/WriteCache.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -51,7 +51,7 @@
import com.bigdata.counters.Instrument;
import com.bigdata.journal.AbstractBufferStrategy;
import com.bigdata.journal.DiskOnlyStrategy;
-import com.bigdata.journal.DiskOnlyStrategy.StoreCounters;
+//import com.bigdata.journal.DiskOnlyStrategy.StoreCounters;
import com.bigdata.rawstore.Bytes;
import com.bigdata.rawstore.IRawStore;
import com.bigdata.rwstore.RWStore;
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractJournal.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractJournal.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractJournal.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -1027,33 +1027,33 @@
}
- case Disk: {
+// case Disk: {
+//
+// /*
+// * Setup the buffer strategy.
+// */
+//
+// fileMetadata = new FileMetadata(file, BufferMode.Disk,
+// useDirectBuffers, initialExtent, maximumExtent, create,
+// isEmptyFile, deleteOnExit, readOnly, forceWrites,
+// offsetBits, //readCacheCapacity, readCacheMaxRecordSize,
+// //readOnly ? null : writeCache,
+// writeCacheEnabled,
+// validateChecksum,
+// createTime, checker, alternateRootBlock);
+//
+// _bufferStrategy = new DiskOnlyStrategy(
+// 0L/* soft limit for maximumExtent */,
+//// minimumExtension,
+// fileMetadata);
+//
+// this._rootBlock = fileMetadata.rootBlock;
+//
+// break;
+//
+// }
- /*
- * Setup the buffer strategy.
- */
-
- fileMetadata = new FileMetadata(file, BufferMode.Disk,
- useDirectBuffers, initialExtent, maximumExtent, create,
- isEmptyFile, deleteOnExit, readOnly, forceWrites,
- offsetBits, //readCacheCapacity, readCacheMaxRecordSize,
- //readOnly ? null : writeCache,
- writeCacheEnabled,
- validateChecksum,
- createTime, checker, alternateRootBlock);
-
- _bufferStrategy = new DiskOnlyStrategy(
- 0L/* soft limit for maximumExtent */,
-// minimumExtension,
- fileMetadata);
-
- this._rootBlock = fileMetadata.rootBlock;
-
- break;
-
- }
-
-// case Disk:
+ case Disk:
case DiskWORM: {
/*
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -7,6 +7,7 @@
import com.bigdata.counters.CounterSet;
import com.bigdata.counters.Instrument;
+import com.bigdata.resources.StoreManager;
import com.bigdata.service.IBigdataFederation;
import com.bigdata.service.IDataService;
@@ -171,16 +172,18 @@
* Delay between attempts reach the remote service (ms).
*/
final long delay = 10L;
-
- /**
- * #of attempts to reach the remote service.
- *
- * Note: delay*maxtries == 1000ms of trying before we give up.
- *
- * If this is not enough, then consider adding an optional parameter giving
- * the time the caller will wait and letting the StoreManager wait longer
- * during startup to discover the timestamp service.
- */
+
+ /**
+ * #of attempts to reach the remote service.
+ * <p>
+ * Note: delay*maxtries == 1000ms of trying before we give up, plus however
+ * long we are willing to wait for service discovery if the problem is
+ * locating the {@link ITransactionService}.
+ * <p>
+ * If this is not enough, then consider adding an optional parameter giving
+ * the time the caller will wait and letting the {@link StoreManager} wait
+ * longer during startup to discover the timestamp service.
+ */
final int maxtries = 100;
/**
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -46,6 +46,7 @@
import com.bigdata.io.DirectBufferPool;
import com.bigdata.io.FileChannelUtility;
import com.bigdata.io.IReopenChannel;
+import com.bigdata.journal.WORMStrategy.StoreCounters;
import com.bigdata.rawstore.Bytes;
import com.bigdata.rawstore.IRawStore;
import com.bigdata.resources.StoreManager.ManagedJournal;
@@ -501,7 +502,7 @@
writeCache.flush();
- storeCounters.ncacheFlush++;
+// storeCounters.ncacheFlush++;
}
@@ -544,551 +545,551 @@
}
- /**
- * Counters for {@link IRawStore} access, including operations that read or
- * write through to the underlying media.
- *
- * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
- *
- * @todo report elapsed time and average latency for force, reopen, and
- * writeRootBlock.
- *
- * @todo counters need to be atomic if we want to avoid the possibility of
- * concurrent <code>x++</code> operations failing to correctly
- * increment <code>x</code> for each request.
- */
- public static class StoreCounters {
-
- /**
- * #of read requests.
- */
- public long nreads;
-
- /**
- * #of read requests that are satisfied by our write cache (vs the
- * OS or disk level write cache).
- */
- public long ncacheRead;
-
- /**
- * #of read requests that read through to the backing file.
- */
- public long ndiskRead;
-
- /**
- * #of bytes read.
- */
- public long bytesRead;
-
- /**
- * #of bytes that have been read from the disk.
- */
- public long bytesReadFromDisk;
-
- /**
- * The size of the largest record read.
- */
- public long maxReadSize;
-
- /**
- * Total elapsed time for reads.
- */
- public long elapsedReadNanos;
-
- /**
- * Total elapsed time checking the disk write cache for records to be
- * read.
- */
- public long elapsedCacheReadNanos;
-
- /**
- * Total elapsed time for reading on the disk.
- */
- public long elapsedDiskReadNanos;
-
- /**
- * #of write requests.
- */
- public long nwrites;
-
- /**
- * #of write requests that are absorbed by our write cache (vs the OS or
- * disk level write cache).
- */
- public long ncacheWrite;
-
- /**
- * #of times the write cache was flushed to disk.
- */
- public long ncacheFlush;
-
- /**
- * #of write requests that write through to the backing file.
- */
- public long ndiskWrite;
-
- /**
- * The size of the largest record written.
- */
- public long maxWriteSize;
-
- /**
- * #of bytes written.
- */
- public long bytesWritten;
-
- /**
- * #of bytes that have been written on the disk.
- */
- public long bytesWrittenOnDisk;
-
- /**
- * Total elapsed time for writes.
- */
- public long elapsedWriteNanos;
-
- /**
- * Total elapsed time writing records into the cache (does not count
- * time to flush the cache when it is full or to write records that do
- * not fit in the cache directly to the disk).
- */
- public long elapsedCacheWriteNanos;
-
- /**
- * Total elapsed time for writing on the disk.
- */
- public long elapsedDiskWriteNanos;
-
- /**
- * #of times the data were forced to the disk.
- */
- public long nforce;
-
- /**
- * #of times the length of the file was changed (typically, extended).
- */
- public long ntruncate;
-
- /**
- * #of times the file has been reopened after it was closed by an
- * interrupt.
- */
- public long nreopen;
-
- /**
- * #of times one of the root blocks has been written.
- */
- public long nwriteRootBlock;
-
- /**
- * Initialize a new set of counters.
- */
- public StoreCounters() {
-
- }
-
- /**
- * Copy ctor.
- * @param o
- */
- public StoreCounters(final StoreCounters o) {
-
- add( o );
-
- }
-
- /**
- * Adds counters to the current counters.
- *
- * @param o
- */
- public void add(final StoreCounters o) {
-
- nreads += o.nreads;
- ncacheRead += o.ncacheRead;
- ndiskRead += o.ndiskRead;
- bytesRead += o.bytesRead;
- bytesReadFromDisk += o.bytesReadFromDisk;
- maxReadSize += o.maxReadSize;
- elapsedReadNanos += o.elapsedReadNanos;
- elapsedCacheReadNanos += o.elapsedCacheReadNanos;
- elapsedDiskReadNanos += o.elapsedDiskReadNanos;
-
- nwrites += o.nwrites;
- ncacheWrite += o.ncacheWrite;
- ncacheFlush += o.ncacheFlush;
- ndiskWrite += o.ndiskWrite;
- maxWriteSize += o.maxWriteSize;
- bytesWritten += o.bytesWritten;
- bytesWrittenOnDisk += o.bytesWrittenOnDisk;
- elapsedWriteNanos += o.elapsedWriteNanos;
- elapsedCacheWriteNanos += o.elapsedCacheWriteNanos;
- elapsedDiskWriteNanos += o.elapsedDiskWriteNanos;
-
- nforce += o.nforce;
- ntruncate += o.ntruncate;
- nreopen += o.nreopen;
- nwriteRootBlock += o.nwriteRootBlock;
-
- }
-
- /**
- * Returns a new {@link StoreCounters} containing the current counter values
- * minus the given counter values.
- *
- * @param o
- *
- * @return
- */
- public StoreCounters subtract(final StoreCounters o) {
-
- // make a copy of the current counters.
- final StoreCounters t = new StoreCounters(this);
-
- // subtract out the given counters.
- t.nreads -= o.nreads;
- t.ncacheRead -= o.ncacheRead;
- t.ndiskRead -= o.ndiskRead;
- t.bytesRead -= o.bytesRead;
- t.bytesReadFromDisk -= o.bytesReadFromDisk;
- t.maxReadSize -= o.maxReadSize;
- t.elapsedReadNanos -= o.elapsedReadNanos;
- t.elapsedCacheReadNanos -= o.elapsedCacheReadNanos;
- t.elapsedDiskReadNanos -= o.elapsedDiskReadNanos;
-
- t.nwrites -= o.nwrites;
- t.ncacheWrite -= o.ncacheWrite;
- t.ncacheFlush -= o.ncacheFlush;
- t.ndiskWrite -= o.ndiskWrite;
- t.maxWriteSize -= o.maxWriteSize;
- t.bytesWritten -= o.bytesWritten;
- t.bytesWrittenOnDisk -= o.bytesWrittenOnDisk;
- t.elapsedWriteNanos -= o.elapsedWriteNanos;
- t.elapsedCacheWriteNanos -= o.elapsedCacheWriteNanos;
- t.elapsedDiskWriteNanos -= o.elapsedDiskWriteNanos;
-
- t.nforce -= o.nforce;
- t.ntruncate -= o.ntruncate;
- t.nreopen -= o.nreopen;
- t.nwriteRootBlock -= o.nwriteRootBlock;
-
- return t;
-
- }
-
- synchronized public CounterSet getCounters() {
-
- if (root == null) {
-
- root = new CounterSet();
-
- // IRawStore API
- {
-
- /*
- * reads
- */
-
- root.addCounter("nreads", new Instrument<Long>() {
- public void sample() {
- setValue(nreads);
- }
- });
-
- root.addCounter("bytesRead", new Instrument<Long>() {
- public void sample() {
- setValue(bytesRead);
- }
- });
-
- root.addCounter("readSecs", new Instrument<Double>() {
- public void sample() {
- final double elapsedReadSecs = (elapsedReadNanos / 1000000000.);
- setValue(elapsedReadSecs);
- }
- });
-
- root.addCounter("bytesReadPerSec",
- new Instrument<Double>() {
- public void sample() {
- final double readSecs = (elapsedReadNanos / 1000000000.);
- final double bytesReadPerSec = (readSecs == 0L ? 0d
- : (bytesRead / readSecs));
- setValue(bytesReadPerSec);
- }
- });
-
- root.addCounter("maxReadSize", new Instrument<Long>() {
- public void sample() {
- setValue(maxReadSize);
- }
- });
-
- /*
- * writes
- */
-
- root.addCounter("nwrites", new Instrument<Long>() {
- public void sample() {
- setValue(nwrites);
- }
- });
-
- root.addCounter("bytesWritten", new Instrument<Long>() {
- public void sample() {
- setValue(bytesWritten);
- }
- });
-
- root.addCounter("writeSecs", new Instrument<Double>() {
- public void sample() {
- final double writeSecs = (elapsedWriteNanos / 1000000000.);
- setValue(writeSecs);
- }
- });
-
- root.addCounter("bytesWrittenPerSec",
- new Instrument<Double>() {
- public void sample() {
- final double writeSecs = (elapsedWriteNanos / 1000000000.);
- final double bytesWrittenPerSec = (writeSecs == 0L ? 0d
- : (bytesWritten / writeSecs));
- setValue(bytesWrittenPerSec);
- }
- });
-
- root.addCounter("maxWriteSize", new Instrument<Long>() {
- public void sample() {
- setValue(maxWriteSize);
- }
- });
-
- }
-
- /*
- * write cache statistics
- */
- {
-
- final CounterSet writeCache = root.makePath("writeCache");
-
- /*
- * read
- */
- writeCache.addCounter("nread", new Instrument<Long>() {
- public void sample() {
- setValue(ncacheRead);
- }
- });
-
- writeCache.addCounter("readHitRate", new Instrument<Double>() {
- public void sample() {
- setValue(nreads == 0L ? 0d : (double) ncacheRead
- / nreads);
- }
- });
-
- writeCache.addCounter("readSecs", new Instrument<Double>() {
- public void sample() {
- setValue(elapsedCacheReadNanos / 1000000000.);
- }
- });
-
- /*
- * write
- */
-
- // #of writes on the write cache.
- writeCache.addCounter("nwrite", new Instrument<Long>() {
- public void sample() {
- setValue(ncacheWrite);
- }
- });
-
- /*
- * % of writes that are buffered vs writing through to the
- * disk.
- *
- * Note: This will be 1.0 unless you are writing large
- * records. Large records are written directly to the disk
- * rather than first into the write cache. When this happens
- * the writeHitRate on the cache can be less than one.
- */
- writeCache.addCounter("writeHitRate", new Instrument<Double>() {
- public void sample() {
- setValue(nwrites == 0L ? 0d : (double) ncacheWrite
- / nwrites);
- }
- });
-
- writeCache.addCounter("writeSecs", new Instrument<Double>() {
- public void sample() {
- setValue(elapsedCacheWriteNanos / 1000000000.);
- }
- });
-
- // #of times the write cache was flushed to the disk.
- writeCache.addCounter("nflush", new Instrument<Long>() {
- public void sample() {
- setValue(ncacheFlush);
- }
- });
-
- }
-
- // disk statistics
- {
- final CounterSet disk = root.makePath("disk");
-
- /*
- * read
- */
-
- disk.addCounter("nreads", new Instrument<Long>() {
- public void sample() {
- setValue(ndiskRead);
- }
- });
-
- disk.addCounter("bytesRead", new Instrument<Long>() {
- public void sample() {
- setValue(bytesReadFromDisk);
- }
- });
-
- disk.addCounter("bytesPerRead", new Instrument<Double>() {
- public void sample() {
- final double bytesPerDiskRead = (ndiskRead == 0 ? 0d
- : (bytesReadFromDisk / (double)ndiskRead));
- setValue(bytesPerDiskRead);
- }
- });
-
- disk.addCounter("readSecs", new Instrument<Double>() {
- public void sample() {
- final double diskReadSecs = (elapsedDiskReadNanos / 1000000000.);
- setValue(diskReadSecs);
- }
- });
-
- disk.addCounter("bytesReadPerSec",
- new Instrument<Double>() {
- public void sample() {
- final double diskReadSecs = (elapsedDiskReadNanos / 1000000000.);
- final double bytesReadPerSec = (diskReadSecs == 0L ? 0d
- : bytesReadFromDisk / diskReadSecs);
- setValue(bytesReadPerSec);
- }
- });
-
- disk.addCounter("secsPerRead", new Instrument<Double>() {
- public void sample() {
- final double diskReadSecs = (elapsedDiskReadNanos / 1000000000.);
- final double readLatency = (diskReadSecs == 0 ? 0d
- : diskReadSecs / ndiskRead);
- setValue(readLatency);
- }
- });
-
- /*
- * write
- */
-
- disk.addCounter("nwrites", new Instrument<Long>() {
- public void sample() {
- setValue(ndiskWrite);
- }
- });
-
- disk.addCounter("bytesWritten", new Instrument<Long>() {
- public void sample() {
- setValue(bytesWrittenOnDisk);
- }
- });
-
- disk.addCounter("bytesPerWrite", new Instrument<Double>() {
- public void sample() {
- final double bytesPerDiskWrite = (ndiskWrite == 0 ? 0d
- : (bytesWrittenOnDisk / (double)ndiskWrite));
- setValue(bytesPerDiskWrite);
- }
- });
-
- disk.addCounter("writeSecs", new Instrument<Double>() {
- public void sample() {
- final double diskWriteSecs = (elapsedDiskWriteNanos / 1000000000.);
- setValue(diskWriteSecs);
- }
- });
-
- disk.addCounter("bytesWrittenPerSec",
- new Instrument<Double>() {
- public void sample() {
- final double diskWriteSecs = (elapsedDiskWriteNanos / 1000000000.);
- final double bytesWrittenPerSec = (diskWriteSecs == 0L ? 0d
- : bytesWrittenOnDisk
- / diskWriteSecs);
- setValue(bytesWrittenPerSec);
- }
- });
-
- disk.addCounter("secsPerWrite", new Instrument<Double>() {
- public void sample() {
- final double diskWriteSecs = (elapsedDiskWriteNanos / 1000000000.);
- final double writeLatency = (diskWriteSecs == 0 ? 0d
- : diskWriteSecs / ndiskWrite);
- setValue(writeLatency);
- }
- });
-
- /*
- * other
- */
-
- disk.addCounter("nforce", new Instrument<Long>() {
- public void sample() {
- setValue(nforce);
- }
- });
-
- disk.addCounter("nextend", new Instrument<Long>() {
- public void sample() {
- setValue(ntruncate);
- }
- });
-
- disk.addCounter("nreopen", new Instrument<Long>() {
- public void sample() {
- setValue(nreopen);
- }
- });
-
- disk.addCounter("rootBlockWrites", new Instrument<Long>() {
- public void sample() {
- setValue(nwriteRootBlock);
- }
- });
-
- }
-
- }
-
- return root;
-
- }
- private CounterSet root;
-
- /**
- * Human readable representation of the counters.
- */
- public String toString() {
-
- return getCounters().toString();
-
- }
-
- }
+// /**
+// * Counters for {@link IRawStore} access, including operations that read or
+// * write through to the underlying media.
+// *
+// * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+// * @version $Id$
+// *
+// * @todo report elapsed time and average latency for force, reopen, and
+// * writeRootBlock.
+// *
+// * @todo counters need to be atomic if we want to avoid the possibility of
+// * concurrent <code>x++</code> operations failing to correctly
+// * increment <code>x</code> for each request.
+// */
+// public static class StoreCounters {
+//
+// /**
+// * #of read requests.
+// */
+// public long nreads;
+//
+// /**
+// * #of read requests that are satisfied by our write cache (vs the
+// * OS or disk level write cache).
+// */
+// public long ncacheRead;
+//
+// /**
+// * #of read requests that read through to the backing file.
+// */
+// public long ndiskRead;
+//
+// /**
+// * #of bytes read.
+// */
+// public long bytesRead;
+//
+// /**
+// * #of bytes that have been read from the disk.
+// */
+// public long bytesReadFromDisk;
+//
+// /**
+// * The size of the largest record read.
+// */
+// public long maxReadSize;
+//
+// /**
+// * Total elapsed time for reads.
+// */
+// public long elapsedReadNanos;
+//
+// /**
+// * Total elapsed time checking the disk write cache for records to be
+// * read.
+// */
+// public long elapsedCacheReadNanos;
+//
+// /**
+// * Total elapsed time for reading on the disk.
+// */
+// public long elapsedDiskReadNanos;
+//
+// /**
+// * #of write requests.
+// */
+// public long nwrites;
+//
+// /**
+// * #of write requests that are absorbed by our write cache (vs the OS or
+// * disk level write cache).
+// */
+// public long ncacheWrite;
+//
+// /**
+// * #of times the write cache was flushed to disk.
+// */
+// public long ncacheFlush;
+//
+// /**
+// * #of write requests that write through to the backing file.
+// */
+// public long ndiskWrite;
+//
+// /**
+// * The size of the largest record written.
+// */
+// public long maxWriteSize;
+//
+// /**
+// * #of bytes written.
+// */
+// public long bytesWritten;
+//
+// /**
+// * #of bytes that have been written on the disk.
+// */
+// public long bytesWrittenOnDisk;
+//
+// /**
+// * Total elapsed time for writes.
+// */
+// public long elapsedWriteNanos;
+//
+// /**
+// * Total elapsed time writing records into the cache (does not count
+// * time to flush the cache when it is full or to write records that do
+// * not fit in the cache directly to the disk).
+// */
+// public long elapsedCacheWriteNanos;
+//
+// /**
+// * Total elapsed time for writing on the disk.
+// */
+// public long elapsedDiskWriteNanos;
+//
+// /**
+// * #of times the data were forced to the disk.
+// */
+// public long nforce;
+//
+// /**
+// * #of times the length of the file was changed (typically, extended).
+// */
+// public long ntruncate;
+//
+// /**
+// * #of times the file has been reopened after it was closed by an
+// * interrupt.
+// */
+// public long nreopen;
+//
+// /**
+// * #of times one of the root blocks has been written.
+// */
+// public long nwriteRootBlock;
+//
+// /**
+// * Initialize a new set of counters.
+// */
+// public StoreCounters() {
+//
+// }
+//
+// /**
+// * Copy ctor.
+// * @param o
+// */
+// public StoreCounters(final StoreCounters o) {
+//
+// add( o );
+//
+// }
+//
+// /**
+// * Adds counters to the current counters.
+// *
+// * @param o
+// */
+// public void add(final StoreCounters o) {
+//
+// nreads += o.nreads;
+// ncacheRead += o.ncacheRead;
+// ndiskRead += o.ndiskRead;
+// bytesRead += o.bytesRead;
+// bytesReadFromDisk += o.bytesReadFromDisk;
+// maxReadSize += o.maxReadSize;
+// elapsedReadNanos += o.elapsedReadNanos;
+// elapsedCacheReadNanos += o.elapsedCacheReadNanos;
+// elapsedDiskReadNanos += o.elapsedDiskReadNanos;
+//
+// nwrites += o.nwrites;
+// ncacheWrite += o.ncacheWrite;
+// ncacheFlush += o.ncacheFlush;
+// ndiskWrite += o.ndiskWrite;
+// maxWriteSize += o.maxWriteSize;
+// bytesWritten += o.bytesWritten;
+// bytesWrittenOnDisk += o.bytesWrittenOnDisk;
+// elapsedWriteNanos += o.elapsedWriteNanos;
+// elapsedCacheWriteNanos += o.elapsedCacheWriteNanos;
+// elapsedDiskWriteNanos += o.elapsedDiskWriteNanos;
+//
+// nforce += o.nforce;
+// ntruncate += o.ntruncate;
+// nreopen += o.nreopen;
+// nwriteRootBlock += o.nwriteRootBlock;
+//
+// }
+//
+// /**
+// * Returns a new {@link StoreCounters} containing the current counter values
+// * minus the given counter values.
+// *
+// * @param o
+// *
+// * @return
+// */
+// public StoreCounters subtract(final StoreCounters o) {
+//
+// // make a copy of the current counters.
+// final StoreCounters t = new StoreCounters(this);
+//
+// // subtract out the given counters.
+// t.nreads -= o.nreads;
+// t.ncacheRead -= o.ncacheRead;
+// t.ndiskRead -= o.ndiskRead;
+// t.bytesRead -= o.bytesRead;
+// t.bytesReadFromDisk -= o.bytesReadFromDisk;
+// t.maxReadSize -= o.maxReadSize;
+// t.elapsedReadNanos -= o.elapsedReadNanos;
+// t.elapsedCacheReadNanos -= o.elapsedCacheReadNanos;
+// t.elapsedDiskReadNanos -= o.elapsedDiskReadNanos;
+//
+// t.nwrites -= o.nwrites;
+// t.ncacheWrite -= o.ncacheWrite;
+// t.ncacheFlush -= o.ncacheFlush;
+// t.ndiskWrite -= o.ndiskWrite;
+// t.maxWriteSize -= o.maxWriteSize;
+// t.bytesWritten -= o.bytesWritten;
+// t.bytesWrittenOnDisk -= o.bytesWrittenOnDisk;
+// t.elapsedWriteNanos -= o.elapsedWriteNanos;
+// t.elapsedCacheWriteNanos -= o.elapsedCacheWriteNanos;
+// t.elapsedDiskWriteNanos -= o.elapsedDiskWriteNanos;
+//
+// t.nforce -= o.nforce;
+// t.ntruncate -= o.ntruncate;
+// t.nreopen -= o.nreopen;
+// t.nwriteRootBlock -= o.nwriteRootBlock;
+//
+// return t;
+//
+// }
+//
+// synchronized public CounterSet getCounters() {
+//
+// if (root == null) {
+//
+// root = new CounterSet();
+//
+// // IRawStore API
+// {
+//
+// /*
+// * reads
+// */
+//
+// root.addCounter("nreads", new Instrument<Long>() {
+// public void sample() {
+// setValue(nreads);
+// }
+// });
+//
+// root.addCounter("bytesRead", new Instrument<Long>() {
+// public void sample() {
+// setValue(bytesRead);
+// }
+// });
+//
+// root.addCounter("readSecs", new Instrument<Double>() {
+// public void sample() {
+// final double elapsedReadSecs = (elapsedReadNanos / 1000000000.);
+// setValue(elapsedReadSecs);
+// }
+// });
+//
+// root.addCounter("bytesReadPerSec",
+// new Instrument<Double>() {
+// public void sample() {
+// final double readSecs = (elapsedReadNanos / 1000000000.);
+// final double bytesReadPerSec = (readSecs == 0L ? 0d
+// : (bytesRead / readSecs));
+// setValue(bytesReadPerSec);
+// }
+// });
+//
+// root.addCounter("maxReadSize", new Instrument<Long>() {
+// public void sample() {
+// setValue(maxReadSize);
+// }
+// });
+//
+// /*
+// * writ...
[truncated message content] |
|
From: <btm...@us...> - 2010-08-11 16:14:07
|
Revision: 3440
http://bigdata.svn.sourceforge.net/bigdata/?rev=3440&view=rev
Author: btmurphy
Date: 2010-08-11 16:14:01 +0000 (Wed, 11 Aug 2010)
Log Message:
-----------
[branch dev-btm]: fixed two minor issues that affected smart proxy deployment; added needed jar files to load balancer classpath in boot-processes.xml, and the Service interface to the uninterestingInterfaces entry in the browser.config file
Modified Paths:
--------------
branches/dev-btm/bigdata-jini/src/java/com/bigdata/boot/config/boot-processes.xml
branches/dev-btm/src/resources/bin/config/browser.config
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/boot/config/boot-processes.xml
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/boot/config/boot-processes.xml 2010-08-09 19:33:34 UTC (rev 3439)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/boot/config/boot-processes.xml 2010-08-11 16:14:01 UTC (rev 3440)
@@ -59,7 +59,7 @@
<javaprop name="java.util.logging.config.file"
value="${bigdata.configDir}/logging/logging.properties"/>
- <property name="java.classpath" value="${bootLauncherClasspath}"/>
+ <property name="java.classpath" value="${bootLauncherClasspath}${:}lib/fastutil.jar${:}lib/dsiutils.jar${:}lib/cweb-extser.jar${:}lib/icu4j.jar${:}lib/ctc_utils.jar${:}lib/lgplutils.jar"/>
<property name="java.app.mainclass"
value="com.bigdata.boot.starter.SingleNonActivatableServiceStarter"/>
<arg value="${bigdata.configDir}/policy/service.policy"/>
Modified: branches/dev-btm/src/resources/bin/config/browser.config
===================================================================
--- branches/dev-btm/src/resources/bin/config/browser.config 2010-08-09 19:33:34 UTC (rev 3439)
+++ branches/dev-btm/src/resources/bin/config/browser.config 2010-08-11 16:14:01 UTC (rev 3440)
@@ -34,6 +34,7 @@
"net.jini.admin.Administrable",
"net.jini.core.constraint.RemoteMethodControl",
"net.jini.id.ReferentUuid",
+ "com.bigdata.service.Service",
"com.bigdata.service.EventReceivingService"
};
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <sgo...@us...> - 2010-08-12 20:09:21
|
Revision: 3443
http://bigdata.svn.sourceforge.net/bigdata/?rev=3443&view=rev
Author: sgossard
Date: 2010-08-12 20:09:13 +0000 (Thu, 12 Aug 2010)
Log Message:
-----------
[dev-btm] : Updating mergeinfo for dev-btm branch to reflect correct changeset eligibility from maven_scaleout branch.
Property Changed:
----------------
branches/dev-btm/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/attr/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/disco/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/
branches/dev-btm/bigdata-perf/
branches/dev-btm/bigdata-perf/lubm/lib/
branches/dev-btm/bigdata-perf/lubm/src/resources/
branches/dev-btm/bigdata-perf/lubm/src/resources/answers (U1)/
branches/dev-btm/bigdata-perf/lubm/src/resources/config/
branches/dev-btm/bigdata-perf/lubm/src/resources/logging/
branches/dev-btm/bigdata-perf/lubm/src/resources/scripts/
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util/
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/
branches/dev-btm/dsi-utils/LEGAL/
branches/dev-btm/dsi-utils/lib/
branches/dev-btm/dsi-utils/src/
branches/dev-btm/dsi-utils/src/test/
branches/dev-btm/dsi-utils/src/test/it/
branches/dev-btm/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/osgi/
Property changes on: branches/dev-btm
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/LEXICON_REFACTOR_BRANCH:2633-3304
/branches/bugfix-btm:2594-3237
/branches/fko:3150-3194
/trunk:2575-2594,2596-2877,2882-2903,2910-3430
+ /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/LEXICON_REFACTOR_BRANCH:2633-3304
/branches/bugfix-btm:2594-3237
/branches/fko:3150-3194
/branches/maven_scaleout:3379-3439,3442
/trunk:2575-2594,2596-2877,2882-2903,2910-3430
Property changes on: branches/dev-btm/bigdata-jini/src/java/com/bigdata/attr
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/bigdata-jini/src/java/com/bigdata/attr:3270-3430
+ /branches/maven_scaleout/bigdata-jini/src/java/com/bigdata/attr:3379-3439,3442
/trunk/bigdata-jini/src/java/com/bigdata/attr:3270-3430
Property changes on: branches/dev-btm/bigdata-jini/src/java/com/bigdata/disco
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/bigdata-jini/src/java/com/bigdata/disco:3270-3430
+ /branches/maven_scaleout/bigdata-jini/src/java/com/bigdata/disco:3379-3439,3442
/trunk/bigdata-jini/src/java/com/bigdata/disco:3270-3430
Property changes on: branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/bigdata-jini/src/java/com/bigdata/util/config:3270-3430
+ /branches/maven_scaleout/bigdata-jini/src/java/com/bigdata/util/config:3379-3439,3442
/trunk/bigdata-jini/src/java/com/bigdata/util/config:3270-3430
Property changes on: branches/dev-btm/bigdata-perf
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/bigdata-perf:3270-3430
+ /branches/maven_scaleout/bigdata-perf:3379-3439,3442
/trunk/bigdata-perf:3270-3430
Property changes on: branches/dev-btm/bigdata-perf/lubm/lib
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/bigdata-perf/lubm/lib:3270-3338,3340-3430
+ /branches/maven_scaleout/bigdata-perf/lubm/lib:3379-3439,3442
/trunk/bigdata-perf/lubm/lib:3270-3338,3340-3430
Property changes on: branches/dev-btm/bigdata-perf/lubm/src/resources
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/bigdata-perf/lubm/src/resources:3270-3338,3340-3430
+ /branches/maven_scaleout/bigdata-perf/lubm/src/resources:3379-3439,3442
/trunk/bigdata-perf/lubm/src/resources:3270-3338,3340-3430
Property changes on: branches/dev-btm/bigdata-perf/lubm/src/resources/answers (U1)
___________________________________________________________________
Modified: svn:mergeinfo
-
+ /branches/maven_scaleout/bigdata-perf/lubm/src/resources/answers (U1):3379-3439,3442
Property changes on: branches/dev-btm/bigdata-perf/lubm/src/resources/config
___________________________________________________________________
Modified: svn:mergeinfo
-
+ /branches/maven_scaleout/bigdata-perf/lubm/src/resources/config:3379-3439,3442
Property changes on: branches/dev-btm/bigdata-perf/lubm/src/resources/logging
___________________________________________________________________
Modified: svn:mergeinfo
-
+ /branches/maven_scaleout/bigdata-perf/lubm/src/resources/logging:3379-3439,3442
Property changes on: branches/dev-btm/bigdata-perf/lubm/src/resources/scripts
___________________________________________________________________
Modified: svn:mergeinfo
-
+ /branches/maven_scaleout/bigdata-perf/lubm/src/resources/scripts:3379-3439,3442
Property changes on: branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/bigdata-rdf/src/java/com/bigdata/rdf/util:3270-3430
+ /branches/maven_scaleout/bigdata-rdf/src/java/com/bigdata/rdf/util:3379-3439,3442
/trunk/bigdata-rdf/src/java/com/bigdata/rdf/util:3270-3430
Property changes on: branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/bench
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/bigdata-sails/src/java/com/bigdata/rdf/sail/bench:3270-3430
+ /branches/maven_scaleout/bigdata-sails/src/java/com/bigdata/rdf/sail/bench:3379-3439,3442
/trunk/bigdata-sails/src/java/com/bigdata/rdf/sail/bench:3270-3430
Property changes on: branches/dev-btm/dsi-utils/LEGAL
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/dsi-utils/LEGAL:3270-3430
+ /branches/maven_scaleout/dsi-utils/LEGAL:3379-3439,3442
/trunk/dsi-utils/LEGAL:3270-3430
Property changes on: branches/dev-btm/dsi-utils/lib
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/dsi-utils/lib:3270-3430
+ /branches/maven_scaleout/dsi-utils/lib:3379-3439,3442
/trunk/dsi-utils/lib:3270-3430
Property changes on: branches/dev-btm/dsi-utils/src
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/dsi-utils/src:3270-3430
+ /branches/maven_scaleout/dsi-utils/src:3379-3439,3442
/trunk/dsi-utils/src:3270-3430
Property changes on: branches/dev-btm/dsi-utils/src/test
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/dsi-utils/src/test:2960-2965,2967-3430
+ /branches/maven_scaleout/dsi-utils/src/test:3379-3439,3442
/trunk/dsi-utils/src/test:2960-2965,2967-3430
Property changes on: branches/dev-btm/dsi-utils/src/test/it
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/dsi-utils/src/test/it:3270-3430
+ /branches/maven_scaleout/dsi-utils/src/test/it:3379-3439,3442
/trunk/dsi-utils/src/test/it:3270-3430
Property changes on: branches/dev-btm/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom:2960-2965,2967-3430
+ /branches/maven_scaleout/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom:3379-3439,3442
/trunk/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom:2960-2965,2967-3430
Property changes on: branches/dev-btm/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom:2960-2965,2967-3430
+ /branches/maven_scaleout/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom:3379-3439,3442
/trunk/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom:2960-2965,2967-3430
Property changes on: branches/dev-btm/osgi
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/osgi:3270-3430
+ /branches/maven_scaleout/osgi:3379-3439,3442
/trunk/osgi:3270-3430
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <btm...@us...> - 2010-08-26 22:13:44
|
Revision: 3463
http://bigdata.svn.sourceforge.net/bigdata/?rev=3463&view=rev
Author: btmurphy
Date: 2010-08-26 22:13:34 +0000 (Thu, 26 Aug 2010)
Log Message:
-----------
[branch dev-btm]: CHECKPOINT- - added smart proxy implementations of the transaction service and shard locator (metadata) service. Also made changes to allow those smart proxy implementations to be started by the ServicesManagerService or BootManager, as well as added new and changed current tests to run against either the smart proxy implementation of those services or the purely remote implemenations. Note that this work is still incomplete, and so includes numerous debug println's and BTM markers; which will be cleaned up in the final commit. Note finally that this changeset also includes hand-merged changes from the trunk for the following changesets: 3434, 3435, 3436, 3437, 3441, 3445, 3447, 3457, 3458, 3459, 3460
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/cache/RingBuffer.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/BufferMode.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/IConcurrencyManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/ILocalTransactionManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/ITransactionService.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/JournalTransactionService.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/TransactionService.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java
branches/dev-btm/bigdata/src/java/com/bigdata/mdi/PartitionLocator.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/CompactingMergeTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/ResourceManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/StoreManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractIndexCache.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractTransactionService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/CommitTimeIndex.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/DataService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/DefaultClientDelegate.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/DefaultServiceFederationDelegate.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/DistributedTransactionService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IBigdataFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IEventReceivingService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IFederationDelegate.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IMetadataService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IServiceShutdown.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IndexCache.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/MetadataIndexCache.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/NoCacheMetadataIndexView.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ShardLocator.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/AbstractScaleOutClientIndexView.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/ClientIndexView.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/RawDataServiceTupleIterator.java
branches/dev-btm/bigdata/src/resources/logging/log4j.properties
branches/dev-btm/bigdata/src/test/com/bigdata/cache/TestRingBuffer.java
branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestAll.java
branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestTransactionService.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/MockLocalTransactionManager.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/MockTransactionService.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/AbstractEmbeddedFederationTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestEventReceiver.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestMove.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestRestartSafe.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/ndx/pipeline/TestMasterTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/BigdataZooDefs.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/BigdataServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/JiniServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/MetadataServerConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServicesManagerConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/TransactionServerConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/JiniServiceProcessHelper.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/EmbeddedLoadBalancer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/AdminProxy.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/Constants.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/PrivateInterface.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/ServiceProxy.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/logging.properties
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/shardlocator.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/process/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/JiniFederation.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/LoadBalancerServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/MetadataServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/lookup/LoadBalancerClient.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/lookup/TransactionServiceClient.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/ServiceMap.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/DumpFederation.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/JiniServicesHelper.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/AdminProxy.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/Constants.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/PrivateInterface.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/ServiceProxy.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/config/logging.properties
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/Util.java
branches/dev-btm/bigdata-jini/src/resources/config/bigdataStandaloneTesting.config
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/DestroyTransactionService.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/TestServiceConfigurationMonitoring.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/TestServiceStarter.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/config/TestServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/config/testfed.config
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/testfed.config
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/master/TestMappedRDFDataLoadMaster.config
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPO.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestOptionals.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/TestLocalTripleStoreTransactionSemantics.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java
branches/dev-btm/src/resources/bin/config/browser.config
branches/dev-btm/src/resources/bin/pstart
branches/dev-btm/src/resources/config/bigdataCluster.config
Added Paths:
-----------
branches/dev-btm/bigdata/src/java/com/bigdata/service/ShardManagement.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/TxState.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/EmbeddedIndexStore.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/EmbeddedShardLocator.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/lookup/ShardLocatorClient.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/EmbeddedTransactionService.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/TestServiceConfigurationMonitoringRemote.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/TestServiceStarterRemote.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/config/TestServiceConfigurationRemote.java
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/cache/RingBuffer.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/cache/RingBuffer.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/cache/RingBuffer.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -154,7 +154,7 @@
public boolean add(final T ref) throws IllegalStateException {
if (ref == null)
- throw new IllegalArgumentException();
+ throw new NullPointerException();
beforeOffer( ref );
@@ -178,7 +178,7 @@
public boolean offer(final T ref) {
if (ref == null)
- throw new IllegalArgumentException();
+ throw new NullPointerException();
beforeOffer( ref );
@@ -387,12 +387,12 @@
if (index < 0 || index >= size)
throw new IllegalArgumentException();
- if (index + 1 == size) {
-
- // remove the LRU position.
- return remove();
-
- }
+// if (index + 1 == size) {
+//
+// // remove the LRU position.
+// return remove();
+//
+// }
/*
* Otherwise we are removing some non-LRU element.
@@ -409,7 +409,7 @@
for (;;) {
- int nexti = (i + 1) % capacity; // update index.
+ final int nexti = (i + 1) % capacity; // update index.
if (nexti != head) {
@@ -491,10 +491,9 @@
*/
final public boolean scanHead(final int nscan, final T ref) {
- assert nscan > 0;
-// if (nscan <= 0)
-// throw new IllegalArgumentException();
-//
+ if (nscan <= 0)
+ throw new IllegalArgumentException();
+
if (ref == null)
throw new IllegalArgumentException();
@@ -581,6 +580,9 @@
public boolean contains(final Object ref) {
+ if (ref == null)
+ throw new NullPointerException();
+
// MRU to LRU scan.
for (int n = 0, i = tail; n < size; n++) {
@@ -601,7 +603,8 @@
throw new NullPointerException();
if (c == this)
- throw new IllegalArgumentException();
+ return true;
+// throw new IllegalArgumentException();
for( Object e : c ) {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -178,7 +178,7 @@
* <p>
* Note: delay*maxtries == 1000ms of trying before we give up, plus however
* long we are willing to wait for service discovery if the problem is
- * locating the {@link ITransactionService}.
+ * locating the transaction service.
* <p>
* If this is not enough, then consider adding an optional parameter giving
* the time the caller will wait and letting the {@link StoreManager} wait
@@ -203,7 +203,8 @@
try {
- final ITransactionService transactionService = getTransactionService();
+//BTM final ITransactionService transactionService = getTransactionService();
+final TransactionService transactionService = getTransactionService();
if (transactionService == null) {
@@ -261,7 +262,8 @@
try {
- final ITransactionService transactionService = getTransactionService();
+//BTM final ITransactionService transactionService = getTransactionService();
+final TransactionService transactionService = getTransactionService();
if (transactionService == null) {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractTask.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractTask.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractTask.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -539,6 +539,8 @@
*/
synchronized final public ILocalBTreeView getIndex(final String name) {
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: ENTERED [name="+name+"]\n");
if (name == null) {
// @todo change to IllegalArgumentException for API consistency?
@@ -548,9 +550,13 @@
// validate that this is a declared index.
assertResource(name);
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: VALIDATED [name="+name+"]\n");
// verify still running.
assertRunning();
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: RUNNING [name="+name+"]\n");
/*
* Test the named index cache first.
@@ -560,10 +566,14 @@
final ILocalBTreeView index = indexCache.get(name);
if (index != null) {
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: INDEX NOT NULL ---> return\n");
// Cached value.
return index;
-
+}else{
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: NULL INDEX ---> continue\n");
}
}
@@ -573,6 +583,8 @@
final StaleLocatorReason reason = resourceManager.getIndexPartitionGone(name);
if (reason != null) {
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: STALE LOCATOR\n");
throw new StaleLocatorException(name, reason);
@@ -582,6 +594,8 @@
final Entry entry = n2a.get(name);
if (entry == null) {
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: NO SUCH INDEX ---> continue\n");
// index did not exist at that time.
throw new NoSuchIndexException(name);
@@ -599,11 +613,15 @@
// the unisolated name2Addr object.
final Name2Addr name2Addr = resourceManager.getLiveJournal()._getName2Addr();
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: getName2Addr\n");
synchronized (name2Addr) {
// recover from unisolated index cache.
btree = name2Addr.getIndexCache(name);
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: btree = name2Addr.getIndexCache\n");
if (btree == null) {
@@ -613,15 +631,23 @@
entry.checkpointAddr,//
false// readOnly
);
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: RE-LOADED btree from store\n");
// set the lastCommitTime on the index.
btree.setLastCommitTime(entry.commitTime);
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: SET LAST COMMIT TIME\n");
// add to the unisolated index cache (must not exist).
name2Addr.putIndexCache(name, btree, false/* replace */);
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: ADD TO INDEX CACHE\n");
if(resourceManager instanceof ResourceManager) {
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: SET BTREE COUNTERS\n");
btree
.setBTreeCounters(((ResourceManager) resourceManager)
.getIndexCounters(name));
@@ -634,9 +660,13 @@
try {
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: RETURN UNISOLATED INDEX VIEW\n");
return getUnisolatedIndexView(name, btree);
} catch (NoSuchStoreException ex) {
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: NO SUCH STORE EXCEPTION\n");
/*
* Add a little more information to the stack trace.
@@ -648,9 +678,13 @@
} else {
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: resourceManager.getIndex [name="+name+", timestamp="+timestamp+"]\n");
final ILocalBTreeView tmp = resourceManager.getIndex(name, timestamp);
if (tmp == null) {
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: tmp == null ---> NO SUCH INDEX EXCEPTION\n");
// Presume client has made a bad request
throw new NoSuchIndexException(name + ", timestamp="
@@ -664,8 +698,12 @@
* we can hold onto it for the duration of the operation.
*/
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: PUT INDEX INTO HARD REFERENCE CACHE\n");
indexCache.put(name, tmp);
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: EXIT ---> return cahced ILocalBTreeView\n");
return tmp;
}
@@ -1915,6 +1953,8 @@
}
public IIndex getIndex(String name) {
+//BTM
+log.warn("\n*** AbstractTask#DelegateTask.getIndex: delegate.getIndex [name="+name+"]\n");
return delegate.getIndex(name);
@@ -2215,6 +2255,8 @@
try {
+//BTM
+log.warn("\n*** AbstractTask.#IsolatedActionJournal: AbstractTask.this.getIndex [name="+name+"]\n");
return AbstractTask.this.getIndex(name);
} catch(NoSuchIndexException ex) {
@@ -2233,6 +2275,8 @@
*/
public IIndex getIndex(String name, long timestamp) {
+//BTM
+log.warn("\n*** AbstractTask#IsolatedActionJournal: getIndex [name="+name+", timestamp="+timestamp+"]\n");
if (timestamp == ITx.UNISOLATED) {
return getIndex(name);
@@ -2532,6 +2576,8 @@
* Note: Does not allow access to {@link ITx#UNISOLATED} indices.
*/
public IIndex getIndex(String name, long timestamp) {
+//BTM
+log.warn("\n*** AbstractTask#ReadOnlyJournal: getIndex [name="+name+", timestamp="+timestamp+"]\n");
if (timestamp == ITx.UNISOLATED)
throw new UnsupportedOperationException();
@@ -2562,6 +2608,8 @@
* {@link ITx#UNISOLATED} index.
*/
public IIndex getIndex(String name) {
+//BTM
+log.warn("\n*** AbstractTask#ReadOnlyJournal: getIndex [name="+name+"] >>> UnsupportedOperationException\n");
throw new UnsupportedOperationException();
@@ -2884,6 +2932,8 @@
}
public IIndex getIndex(String name, long timestamp) {
+//BTM
+log.warn("\n*** AbstractTask#DelegateIndexManager: getIndex [name="+name+", timestamp="+timestamp+"]\n");
return delegate.getIndex(name, timestamp);
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/BufferMode.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/BufferMode.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/BufferMode.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -59,15 +59,16 @@
Transient(false/* stable */, true/* fullyBuffered */),
/**
+ * <strong>This mode is not being actively developed and should not be used
+ * outside of unit tests.</strong>
* <p>
- * A direct buffer is allocated for the file image. Writes are applied
- * to the buffer. The buffer tracks dirty slots regardless of the
- * transaction that wrote them and periodically writes dirty slots
- * through to disk. On commit, any dirty index or allocation nodes are
- * written onto the buffer and all dirty slots on the buffer. Dirty
- * slots in the buffer are then synchronously written to disk, the
- * appropriate root block is updated, and the file is (optionally)
- * flushed to disk.
+ * A direct buffer is allocated for the file image. Writes are applied to
+ * the buffer. The buffer tracks dirty slots regardless of the transaction
+ * that wrote them and periodically writes dirty slots through to disk. On
+ * commit, any dirty index or allocation nodes are written onto the buffer
+ * and all dirty slots on the buffer. Dirty slots in the buffer are then
+ * synchronously written to disk, the appropriate root block is updated, and
+ * the file is (optionally) flushed to disk.
* </p>
* <p>
* This option offers wires an image of the journal file into memory and
@@ -79,6 +80,9 @@
Direct(true/* stable */, true/* fullyBuffered */),
/**
+ * <strong>This mode is not being actively developed and should not be used
+ * outside of unit tests. Memory mapped IO has the fatal weakness under Java
+ * that you can not reliably close or extend the backing file.</strong>
* <p>
* A memory-mapped buffer is allocated for the file image. Writes are
* applied to the buffer. Reads read from the buffer. On commit, the map is
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/IConcurrencyManager.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/IConcurrencyManager.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/IConcurrencyManager.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -53,12 +53,7 @@
* The client side of the transaction manager.
*/
public ILocalTransactionManager getTransactionManager();
-
-// /**
-// * The server side of the transaction manager.
-// */
-// public ITransactionManager getTransactionService();
-
+
/**
* The object used to manage local resources.
*/
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/ILocalTransactionManager.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/ILocalTransactionManager.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/ILocalTransactionManager.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -32,21 +32,21 @@
/**
* Interface for managing local transaction state (the client side of the
- * {@link ITransactionService}).
+ * transaction service).
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
*
* @todo we don't really need an interface for this as there is only one impl.
*/
-public interface ILocalTransactionManager extends
- /* ITransactionManager, */IServiceShutdown {
+public interface ILocalTransactionManager extends IServiceShutdown {
/**
* The server side of the transaction manager (possibly remote, in which
* case this may require the service to be discovered).
*/
- public ITransactionService getTransactionService();
+//BTM public ITransactionService getTransactionService();
+public TransactionService getTransactionService();
/**
* Return the local state for a transaction.
@@ -61,13 +61,13 @@
public ITx getTx(final long tx);
/**
- * Return the next timestamp from the {@link ITransactionService}.
+ * Return the next timestamp from the transaction service.
* <p>
* Note: This method is "robust" and will "retry"
- * {@link ITransactionService#nextTimestamp()} several times before giving
+ * {@link TransactionService#nextTimestamp()} several times before giving
* up.
*
- * @return The next timestamp assigned by the {@link ITransactionService}.
+ * @return The next timestamp assigned by the transaction service.
*
* @throws RuntimeException
* if the service can not be resolved or the timestamp can not
@@ -86,7 +86,7 @@
* @param commitTime
* The commit time.
*
- * @see ITransactionService#notifyCommit(long)
+ * @see TransactionService#notifyCommit(long)
*/
public void notifyCommit(long commitTime);
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/ITransactionService.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/ITransactionService.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/ITransactionService.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -36,6 +36,9 @@
import com.bigdata.service.IDataService;
import com.bigdata.service.ITxCommitProtocol;
+//BTM
+import com.bigdata.service.IService;
+
/**
* <p>
* An interface for managing transaction life cycles.
@@ -126,206 +129,6 @@
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
*/
-public interface ITransactionService extends ITimestampService {
+public interface ITransactionService extends TransactionService, IService {
- /**
- * Create a new transaction.
- *
- * @param timestamp
- * The timestamp may be:
- * <ul>
- * <li>A timestamp (GT ZERO), which will result in a
- * read-historical (read-only) transaction that reads from the
- * most recent committed state whose commit timestamp is less
- * than or equal to <i>timestamp</i>.</li>
- * <li>The symbolic constant {@link ITx#READ_COMMITTED} to
- * obtain a read-historical transaction reading from the most
- * recently committed state of the database. The transaction will
- * be assigned a start time corresponding to the most recent
- * commit point of the database and will be a fully isolated
- * read-only view of the state of the database as of that start
- * time. (This is an atomic shorthand for
- * newTx(getLastCommitTime())).</li>
- * <li>{@link ITx#UNISOLATED} for a read-write transaction.</li>
- * </ul>
- *
- * @return The unique transaction identifier.
- *
- * @throws IllegalStateException
- * if the requested timestamp is greater than
- * {@link #getLastCommitTime()}.
- * @throws IllegalStateException
- * if the requested timestamp is for a commit point that is no
- * longer preserved by the database (the resources for that
- * commit point have been released).
- * @throws IOException
- * RMI errors.
- *
- * @todo specialize exception for a timestamp that is no longer preserved
- * and for one that is in the future?
- */
- public long newTx(long timestamp) throws IOException;
-
- /**
- * Request commit of the transaction write set. Committing a read-only
- * transaction is necessary in order to release read locks (this is very
- * fast). If a transaction has a write set, then this method does not return
- * until that write set has been made restart safe or the transaction has
- * failed.
- * <p>
- * The commit of a transaction with a write set on a single
- * {@link IDataService} does not require either {@link ITx#UNISOLATED} tasks
- * or other transactions to wait. The latency for such commits is directly
- * related to the size of the transaction write set.
- * <p>
- * However, the commit of a transaction with writes on more than one
- * {@link IDataService} requires a distributed commit protocol. The
- * distributed commit protocol forces ALL tasks writing on those
- * {@link IDataService}s to wait until the transaction is complete. This is
- * necessary in order to obtain a global commit point that corresponds to
- * the atomic commit state of the transaction (without this we would not
- * have the Atomic property for distributed transaction commits).
- *
- * @param tx
- * The transaction identifier.
- *
- * @return The commit time for the transaction -or- ZERO (0L) if the
- * transaction was read-only or had an empty write set. This commit
- * time identifies a global commit point on the database from which
- * you may read the coherent post-commit state of the transaction.
- *
- * @throws ValidationError
- * if the transaction could not be validated.
- * @throws IllegalStateException
- * if <i>tx</i> is not an active transaction.
- * @throws IOException
- * RMI errors.
- */
- public long commit(long tx) throws ValidationError, IOException;
-
- /**
- * Request abort of the transaction write set.
- *
- * @param tx
- * The transaction identifier.
- *
- * @throws IllegalStateException
- * if <i>tx</i> is not an active transaction.
- * @throws IOException
- * RMI errors.
- */
- public void abort(long tx) throws IOException;
-
- /**
- * Notify the {@link ITransactionService} that a commit has been performed
- * with the given timestamp (which it assigned) and that it should update
- * its lastCommitTime iff the given commitTime is GT its current
- * lastCommitTime.
- * <p>
- * Note: This is used to inform the {@link ITransactionService} of commits
- * that DO NOT involve transaction commits. That is, local unisolated writes
- * on individual {@link IDataService}s in an {@link IBigdataFederation}.
- *
- * @param commitTime
- * The commit time.
- *
- * @throws IOException
- */
- public void notifyCommit(long commitTime) throws IOException;
-
- /**
- * Return the last commitTime reported to the {@link ITransactionService}.
- *
- * @return The last known commit time.
- *
- * @throws IOException
- */
- public long getLastCommitTime() throws IOException;
-
- /**
- * Return the timestamp whose historical data MAY be release. This time is
- * derived from the timestamp of the earliest running transaction MINUS the
- * minimum release age and is updated whenever the earliest running
- * transaction terminates. This value is monotonically increasing. It will
- * never be GT the last commit time. It will never be negative. It MAY be
- * ZERO (0L) and will be ZERO (0L) on startup.
- */
- public long getReleaseTime() throws IOException;
-
- /**
- * An {@link IDataService} MUST invoke this method before permitting an
- * operation isolated by a read-write transaction to execute with access to
- * the named resources (this applies only to distributed databases). The
- * declared resources are used in the commit phase of the read-write tx to
- * impose a partial order on commits. That partial order guarentees that
- * commits do not deadlock in contention for the same resources.
- *
- * @param tx
- * The transaction identifier.
- * @param dataService
- * The {@link UUID} an {@link IDataService} on which the
- * transaction will write.
- * @param resource
- * An array of the named resources which the transaction will use
- * on that {@link IDataService} (this may be different for each
- * operation submitted by that transaction to the
- * {@link IDataService}).
- *
- * @return {@link IllegalStateException} if the transaction is not an active
- * read-write transaction.
- */
- public void declareResources(long tx, UUID dataService, String[] resource)
- throws IOException;
-
- /**
- * Callback by an {@link IDataService} participating in a two phase commit
- * for a distributed transaction. The {@link ITransactionService} will wait
- * until all {@link IDataService}s have prepared. It will then choose a
- * <i>commitTime</i> for the transaction and return that value to each
- * {@link IDataService}.
- * <p>
- * Note: If this method throws ANY exception then the task MUST cancel the
- * commit, discard the local write set of the transaction, and note that the
- * transaction is aborted in its local state.
- *
- * @param tx
- * The transaction identifier.
- * @param dataService
- * The {@link UUID} of the {@link IDataService} which sent the
- * message.
- *
- * @return The assigned commit time.
- *
- * @throws InterruptedException
- * @throws BrokenBarrierException
- * @throws IOException
- * if there is an RMI problem.
- */
- public long prepared(long tx, UUID dataService) throws IOException,
- InterruptedException, BrokenBarrierException;
-
- /**
- * Sent by a task participating in a distributed commit of a transaction
- * when the task has successfully committed the write set of the transaction
- * on the live journal of the local {@link IDataService}. If this method
- * returns <code>false</code> then the distributed commit has failed and
- * the task MUST rollback the live journal to the previous commit point. If
- * the return is <code>true</code> then the distributed commit was
- * successful and the task should halt permitting the {@link IDataService}
- * to return from the {@link ITxCommitProtocol#prepare(long, long)} method.
- *
- * @param tx
- * The transaction identifier.
- * @param dataService
- * The {@link UUID} of the {@link IDataService} which sent the
- * message.
- *
- * @return <code>true</code> if the distributed commit was successfull and
- * <code>false</code> if there was a problem.
- *
- * @throws IOException
- */
- public boolean committed(long tx, UUID dataService) throws IOException,
- InterruptedException, BrokenBarrierException;
-
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/JournalTransactionService.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/JournalTransactionService.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/JournalTransactionService.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -37,6 +37,9 @@
import com.bigdata.service.AbstractTransactionService;
import com.bigdata.service.DataService;
+//BTM
+import com.bigdata.service.TxState;
+
/**
* Implementation for a standalone journal using single-phase commits.
*
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/TransactionService.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/TransactionService.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/TransactionService.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -114,7 +114,7 @@
* services as read-locks are managed solely through the advancement of the
* release time by the transaction manager.
*/
-public interface TransactionService extends Service {
+public interface TransactionService {
/**
* Creates a new transaction.
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -1004,7 +1004,8 @@
* The task threw some other kind of exception.
*/
- log.warn("Task failed: task=" + r);//, t);
+//BTM log.warn("Task failed: task=" + r);//, t);
+log.warn("Task failed: task=" + r, t);
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/mdi/PartitionLocator.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/mdi/PartitionLocator.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/mdi/PartitionLocator.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -185,6 +185,16 @@
}
+ /*
+ * @todo There are some unit tests which depend on this implementation of
+ * equals. However, since the partition locator Id for a given scale out
+ * index SHOULD be immutable, running code can rely on partitionId ==
+ * o.partitionId. Therefore the unit tests should be modified to extract an
+ * "assertSamePartitionLocator" method and rely on that. We could then
+ * simplify this method to just test the partitionId. That would reduce the
+ * effort when maintaining hash tables based on the PartitionLocator since
+ * we would not be comparing the keys, UUIDs, etc.
+ */
public boolean equals(final Object o) {
if (this == o)
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -61,13 +61,17 @@
import com.bigdata.service.EventResource;
import com.bigdata.service.EventType;
import com.bigdata.service.IDataService;
-import com.bigdata.service.MetadataService;
+//BTM import com.bigdata.service.MetadataService;
import com.bigdata.util.InnerCause;
import com.bigdata.util.concurrent.DaemonThreadFactory;
import com.bigdata.util.concurrent.LatchedExecutor;
//BTM
+import com.bigdata.metadata.EmbeddedShardLocator;
+import com.bigdata.service.IDataService;
import com.bigdata.service.LoadBalancer;
+import com.bigdata.service.ShardLocator;
+import com.bigdata.service.ShardManagement;
/**
* This class examines the named indices defined on the journal identified by
@@ -116,7 +120,7 @@
* When each task is finished, it submits and awaits the completion of an
* {@link AbstractAtomicUpdateTask}. The atomic update tasks use
* {@link ITx#UNISOLATED} operations on the live journal to make atomic updates
- * to the index partition definitions and to the {@link MetadataService} and/or
+ * to the index partition definitions and to the shard locator service and/or
* a remote data service where necessary.</dd>
* </dl>
* <p>
@@ -634,7 +638,7 @@
if (vmd.mergePriority > 0 || forceCompactingMerges) {
- if(forceCompactingMerges && vmd.getAction().equals(OverflowActionEnum.Copy)) {
+ if(forceCompactingMerges && OverflowActionEnum.Copy.equals(vmd.getAction())) {
vmd.clearCopyAction();
@@ -1190,13 +1194,38 @@
keys, null/*vals*/);
final ResultBuffer resultBuffer;
try {
- resultBuffer = (ResultBuffer) resourceManager
- .getFederation().getMetadataService()
- .submit(
- TimestampUtility.asHistoricalRead(lastCommitTime),
- MetadataService
- .getMetadataIndexName(scaleOutIndexName),
- op).get();
+//BTM resultBuffer = (ResultBuffer) resourceManager
+//BTM .getFederation().getMetadataService()
+//BTM .submit(
+//BTM TimestampUtility.asHistoricalRead(lastCommitTime),
+//BTM MetadataService
+//BTM .getMetadataIndexName(scaleOutIndexName),
+//BTM op).get();
+
+ShardLocator mds = (resourceManager.getFederation()).getMetadataService();
+if(mds == null) {
+ log.error("AsynchronousOverflowTask.chooseJoins: null shard locator (metadata) service - could not locate rightSiblings [index="+scaleOutIndexName+"]");
+ continue;
+}
+if(mds instanceof IDataService) {
+ IDataService remoteShardMgr = (IDataService)mds;
+ resultBuffer =
+ (ResultBuffer) remoteShardMgr.submit
+ (TimestampUtility.asHistoricalRead(lastCommitTime),
+ EmbeddedShardLocator.getMetadataIndexName(scaleOutIndexName),
+ op).get();
+} else if(mds instanceof ShardManagement) {
+ ShardManagement shardMgr = (ShardManagement)mds;
+ resultBuffer =
+ (ResultBuffer) shardMgr.submit
+ (TimestampUtility.asHistoricalRead(lastCommitTime),
+ EmbeddedShardLocator.getMetadataIndexName(scaleOutIndexName),
+ op).get();
+} else {
+ log.error("AsynchronousOverflowTask.chooseJoins: shard locator (metadata) service wrong type [type="+mds.getClass()+"] - could not locate rightSiblings [index="+scaleOutIndexName+"]");
+ continue;
+}
+
} catch (Exception e) {
log.error("Could not locate rightSiblings: index="
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/resources/CompactingMergeTask.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/resources/CompactingMergeTask.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/resources/CompactingMergeTask.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -33,7 +33,11 @@
import com.bigdata.service.ndx.ClientIndexView;
//BTM
+import com.bigdata.metadata.EmbeddedShardLocator;
+import com.bigdata.service.IDataService;
import com.bigdata.service.LoadBalancer;
+import com.bigdata.service.ShardLocator;
+import com.bigdata.service.ShardManagement;
/**
* Task builds an {@link IndexSegment} from the fused view of an index partition
@@ -749,13 +753,37 @@
final ResultBuffer resultBuffer;
try {
- resultBuffer = (ResultBuffer) resourceManager.getFederation()
- .getMetadataService().submit(
- TimestampUtility.asHistoricalRead(lastCommitTime),
- MetadataService
- .getMetadataIndexName(scaleOutIndexName),
+//BTM resultBuffer = (ResultBuffer) resourceManager.getFederation()
+//BTM .getMetadataService().submit(
+//BTM TimestampUtility.asHistoricalRead(lastCommitTime),
+//BTM MetadataService
+//BTM .getMetadataIndexName(scaleOutIndexName),
+//BTM op).get();
+
+ShardLocator mds = (resourceManager.getFederation()).getMetadataService();
+if(mds == null) {
+ log.error("CompactingMergeTask.getRightSiblingLocator: null shard locator (metadata) service - could not locate rightSiblings [index="+scaleOutIndexName+"]");
+ return null;
+}
+if(mds instanceof IDataService) {
+ IDataService remoteShardMgr = (IDataService)mds;
+ resultBuffer =
+ (ResultBuffer) remoteShardMgr.submit
+ (TimestampUtility.asHistoricalRead(lastCommitTime),
+ EmbeddedShardLocator.getMetadataIndexName(scaleOutIndexName),
op).get();
-
+} else if(mds instanceof ShardManagement) {
+ ShardManagement shardMgr = (ShardManagement)mds;
+ resultBuffer =
+ (ResultBuffer) shardMgr.submit
+ (TimestampUtility.asHistoricalRead(lastCommitTime),
+ EmbeddedShardLocator.getMetadataIndexName(scaleOutIndexName),
+ op).get();
+} else {
+ log.error("CompactingMergeTask.getRightSiblingLocator: shard locator (metadata) service wrong type [type="+mds.getClass()+"] - could not locate rightSiblings [index="+scaleOutIndexName+"]");
+ return null;
+}
+
} catch (Exception e) {
log.error("Could not locate rightSiblings: index="
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/resources/ResourceManager.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/resources/ResourceManager.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/resources/ResourceManager.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -44,10 +44,13 @@
import com.bigdata.mdi.LocalPartitionMetadata;
import com.bigdata.service.AbstractFederation;
import com.bigdata.service.DataService;
-import com.bigdata.service.IMetadataService;
+//BTM import com.bigdata.service.IMetadataService;
import com.bigdata.service.MetadataService;
import com.bigdata.util.ReverseLongComparator;
+//BTM
+import com.bigdata.service.ShardLocator;
+
/**
* The {@link ResourceManager} has broad responsibility for journal files, index
* segment files, maintaining index views during overflow processing, and
@@ -725,7 +728,7 @@
/**
* Requests a new index partition identifier from the
- * {@link MetadataService} for the specified scale-out index (RMI).
+ * shard locator service for the specified scale-out index (RMI).
*
* @return The new index partition identifier.
*
@@ -734,7 +737,8 @@
*/
public int nextPartitionId(final String scaleOutIndexName) {
- final IMetadataService mds = getFederation().getMetadataService();
+//BTM final IMetadataService mds = getFederation().getMetadataService();
+final ShardLocator mds = getFederation().getMetadataService();
if (mds == null) {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/resources/StoreManager.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/resources/StoreManager.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/resources/StoreManager.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -81,7 +81,7 @@
import com.bigdata.journal.IResourceLockService;
import com.bigdata.journal.IResourceManager;
import com.bigdata.journal.IRootBlockView;
-import com.bigdata.journal.ITransactionService;
+//BTM import com.bigdata.journal.ITransactionService;
import com.bigdata.journal.ITx;
import com.bigdata.journal.Name2Addr;
import com.bigdata.journal.TemporaryStore;
@@ -107,6 +107,9 @@
import com.bigdata.sparse.SparseRowStore;
import com.bigdata.util.concurrent.DaemonThreadFactory;
+//BTM
+import com.bigdata.journal.TransactionService;
+
/**
* Class encapsulates logic for managing the store files (journals and index
* segments), including the logic to compute the effective release time for the
@@ -724,8 +727,9 @@
*/
protected void assertNotOpen() {
- if (isOpen())
+ if (isOpen()) {
throw new IllegalStateException();
+ }
}
@@ -1177,7 +1181,6 @@
// Note: dataDir is _canonical_
final File dataDir;
try {
-
final String val = properties.getProperty(Options.DATA_DIR);
if (val == null) {
@@ -1442,12 +1445,35 @@
*/
throw new UnsupportedOperationException();
}
- while (true) {
- if (fed.getTransactionService() != null) {
- break;
- }
- log.warn("Waiting for transaction service discovery");
- }
+//BTM while (true) {
+//BTM if (fed.getTransactionService() != null) {
+//BTM break;
+//BTM }
+//BTM log.warn("Waiting for transaction service discovery");
+//BTM }
+//BTM - BEGIN
+//while (true) {
+boolean discoveredTxnSrvc = false;
+for(int i=0;i<60;i++) {
+ if (fed.getTransactionService() != null) {
+ discoveredTxnSrvc = true;
+ break;
+ }
+ try { Thread.sleep(1000L); } catch(InterruptedException ie) { }
+ log.warn("Waiting for transaction service discovery");
+}
+if(discoveredTxnSrvc) {
+ log.warn("\n>>>> discovered transaction service ---- YES\n");
+} else {
+ log.warn("\n>>>> discovered transaction service ---- NO\n");
+ StackTraceElement[] e = (Thread.currentThread()).getStackTrace();
+ StringBuffer buf = new StringBuffer(" "+(e[0]).toString()+"\n");
+ for(int i=1;i<e.length;i++) {
+ buf.append(" "+(e[i]).toString()+"\n");
+ }
+ String stackTrace = buf.toString();
+ log.warn("\n"+stackTrace+"\n");
+}
} catch (UnsupportedOperationException ex) {
log.warn("Federation not available - running in test case?");
}
@@ -1926,6 +1952,7 @@
* Note: clear before we clear [starting] or the
* StoreManager#isRunning() could report true.
*/
+
this.open.set(false);
startupService.shutdownNow();
@@ -3025,7 +3052,7 @@
}
/**
- * Report the next timestamp assigned by the {@link ITransactionService}.
+ * Report the next timestamp assigned by the transaction service.
*/
protected long nextTimestamp() {
@@ -3193,6 +3220,8 @@
* [lastOverflowTime] here to have the same semantics.
*/
final long lastCommitTime = getLiveJournal().getRootBlockView().getLastCommitTime();
+//BTM
+log.warn("\n*** StoreManager.purgeOldResources: lastCommitTime="+lastCommitTime+"\n");
if (lastCommitTime == 0L) {
@@ -3226,12 +3255,15 @@
try {
- final ITransactionService txService = fed
+//BTM final ITransactionService txService = fed
+final TransactionService txService = fed
.getTransactionService();
if (txService != null) {
this.releaseTime = txService.getReleaseTime();
+//BTM
+log.warn("\n*** StoreManager.purgeOldResources: this.releaseTime="+this.releaseTime+"\n");
} else {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -764,8 +764,12 @@
assertOpen();
- return getIndexCache().getIndex(name, timestamp);
-
+//BTM return getIndexCache().getIndex(name, timestamp);
+
+AbstractIndexCache cache = getIndexCache();
+com.bigdata.btree.IRangeQuery index = cache.getIndex(name, timestamp);
+log.warn("\n>>>>AbstractFederation.getIndex: name="+name+", timestamp="+timestamp+", indexCache="+cache+", index="+index+"\n");
+return (IClientIndex)index;
}
public void dropIndex(String name) {
@@ -944,6 +948,16 @@
}
+//BTM - BEGIN
+ public void serviceJoin(final Service service, final UUID serviceUUID) {
+ if (!isOpen()) return;
+ if (log.isInfoEnabled()) {
+ log.info("service=" + service + ", serviceUUID" + serviceUUID);
+ }
+ client.getDelegate().serviceJoin(service, serviceUUID);
+ }
+//BTM - END
+
/**
* Delegated. {@inheritDoc}
*/
@@ -1432,12 +1446,21 @@
if (service == null) {
+//BTM
+ShardLocator mds = null;
if (mdsUUID == null) {
try {
- mdsUUID = fed.getMetadataService().getServiceUUID();
-
+//BTM mdsUUID = fed.getMetadataService().getServiceUUID();
+mds = fed.getMetadataService();
+if( mds instanceof IService ) {
+ mdsUUID = ((IService)mds).getServiceUUID();
+} else if( mds instanceof Service ) {
+ mdsUUID = ((Service)mds).getServiceUUID();
+} else {
+ log.warn("wrong type for shard locator service ["+mds.getClass()+"]");
+}
} catch (IOException ex) {
throw new RuntimeException(ex);
@@ -1456,7 +1479,9 @@
* want the service.
*/
- service = fed.getMetadataService();
+//BTM service = fed.getMetadataService();
+if( (mds != null) && (mds instanceof IDataService) ) service = (IDataService)mds;
+
}
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractIndexCache.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractIndexCache.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractIndexCache.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -61,7 +61,7 @@
protected AbstractIndexCache(final int capacity, final long timeout) {
// indexCache = new WeakValueCache<NT, T>(new LRUCache<NT, T>(capacity));
-
+log.warn("\nAbstractIndexCache.constructor >>> CACHE: capacity="+capacity+", TIMEOUT="+timeout+", to nanos = "+TimeUnit.MILLISECONDS.toNanos(timeout)+"\n");
indexCache = new ConcurrentWeakValueCacheWithTimeout<NT, T>(capacity,
TimeUnit.MILLISECONDS.toNanos(timeout));
@@ -112,6 +112,7 @@
return ndx;
}
+log.warn("\nAbstractIndexCache.constructor >>> 1. ndx == NULL\n");
/*
* Acquire a lock for the index name and timestamp. This allows
@@ -125,8 +126,10 @@
ndx = indexCache.get(nt);
if (ndx == null) {
+log.warn("\nAbstractIndexCache.constructor >>> 2. ndx == NULL\n");
if ((ndx = newView(name, timestamp)) == null) {
+log.warn("\nAbstractIndexCache.constructor >>> 3. newView -- indx == NULL\n");
if (INFO)
log.info("name=" + name + " @ "
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -228,8 +228,12 @@
assertOpen();
- return getMetadataIndexCache().getIndex(name, timestamp);
-
+//BTM return getMetadataIndexCache().getIndex(name, timestamp);
+
+MetadataIndexCache cache = getMetadataIndexCache();
+IMetadataIndex index = cache.getIndex(name, timestamp);
+log.warn("\n>>>>AbstractScaleOutFederation.getMetadataIndex: name="+name+", timestamp="+timestamp+", metadataIndexCache="+cache+", metadataIndex="+index+"\n");
+return index;
}
/**
@@ -377,7 +381,7 @@
}
/**
- * Await the availability of an {@link IMetadataService} and the specified
+ * Await the availability of a shard locator service and the specified
* minimum #of {@link IDataService}s.
*
* @param minDataServices
@@ -398,7 +402,7 @@
* if the client is not connected to the federation.
* @throws InterruptedException
* if this thread is interrupted while awaiting the availability
- * of the {@link MetadataService} or the specified #of
+ * of the shard locator service or the specified #of
* {@link DataService}s.
* @throws TimeoutException
* If a timeout occurs.
@@ -432,7 +436,8 @@
int ntries = 0;
// updated each time through the loop.
- IMetadataService metadataService = null;
+//BTM IMetadataService metadataService = null;
+ShardLocator metadataService = null;
// updated each time through the loop.
UUID[] dataServiceUUIDs = null;
@@ -524,7 +529,7 @@
* to force the {@link DataService} to its minimum possible
* footprint.
*
- * @todo when overflow processing is enabled for the {@link MetadataService}
+ * @todo when overflow processing is enabled for the shard locator service
* we will have to modify this to also trigger overflow for those
* services.
*/
@@ -616,6 +621,22 @@
private final boolean truncateJournal;
+//BTM - BEGIN
+private final ShardLocator metadataService;
+public PurgeResourcesTask(final ShardLocator metadataService, final boolean truncateJournal) {
+ if (metadataService == null) {
+ throw new IllegalArgumentException
+ ("null shard locator service");
+ }
+
+ this.dataService = null;
+ this.metadataService = metadataService;
+
+ this.truncateJournal = truncateJournal;
+
+}
+//BTM - END
+
public PurgeResourcesTask(final IDataService dataService,
final boolean truncateJournal) {
@@ -623,6 +644,8 @@
throw new IllegalArgumentException();
this.dataService = dataService;
+//BTM
+this.metadataService = null;
this.truncateJournal = truncateJournal;
@@ -630,6 +653,8 @@
public Void call() throws Exception {
+//BTM
+if(dataService != null) {
if (log.isInfoEnabled())
log.info("dataService: " + dataService.getServiceName());
@@ -639,6...
[truncated message content] |
|
From: <btm...@us...> - 2010-10-04 18:57:20
|
Revision: 3722
http://bigdata.svn.sourceforge.net/bigdata/?rev=3722&view=rev
Author: btmurphy
Date: 2010-10-04 18:57:09 +0000 (Mon, 04 Oct 2010)
Log Message:
-----------
[branch dev-btm]: CHECKPOINT - phase 1 of the smart proxy implementation of the shard (data) service; which still includes numerous debug println's and BTM markers that will be cleaned up during phase 2. Also made appropriate additional changes to smart proxy implementations of the load balancer, the shard locator (metadata), and transaction services (from previous changesets), as required by the new smart proxy shard service. With these changes, either the ServicesManagerService mechanism or the BootManager mechanism can be used to start those service implementations. Note that this changeset also includes hand-merged changes from the trunk for the following changesets: 3481
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IRangeQuery.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexMetadata.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegment.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentStore.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/ResultSet.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/proc/AbstractKeyArrayIndexProcedureConstructor.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/proc/IIndexProcedure.java
branches/dev-btm/bigdata/src/java/com/bigdata/config/Configuration.java
branches/dev-btm/bigdata/src/java/com/bigdata/counters/CounterSet.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/ConcurrencyManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/IBTreeManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/IResourceManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/ITransactionService.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/JournalTransactionService.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/RegisterIndexTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/Tx.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java
branches/dev-btm/bigdata/src/java/com/bigdata/mdi/IMetadataIndex.java
branches/dev-btm/bigdata/src/java/com/bigdata/mdi/IPartitionMetadata.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/accesspath/AbstractAccessPath.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/accesspath/IAccessPath.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/locator/DefaultResourceLocator.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/AbstractStepTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/IJoinNexus.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/MutationTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/ProgramTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/QueryTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinMasterTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/JoinTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/JoinTaskFactoryTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/AbstractResourceManagerTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/CompactingMergeTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/IncrementalBuildTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/IndexManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/JoinIndexPartitionTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/MoveResult.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/MoveTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/OverflowManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/OverflowMetadata.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/ResourceManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/ScatterSplitTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/SplitIndexPartitionTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/SplitTailTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/StoreManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractIndexCache.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractRoundRobinServiceLoadHelper.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractScaleOutClient.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractServiceLoadHelper.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractTransactionService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/CacheOnceMetadataIndex.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/CachingMetadataIndex.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ClientService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/DataService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/DataServiceCallable.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/DistributedTransactionService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/Event.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IBigdataClient.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IBigdataFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IClientService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IDataService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IDataServiceCallable.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IRemoteExecutor.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IServiceLoadHelper.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ITxCommitProtocol.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ListIndicesTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/LoadBalancerService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/MetadataIndexCache.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/MetadataService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/NoCacheMetadataIndexView.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ShardManagement.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ShardService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/AbstractDataServiceProcedureTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/AbstractScaleOutClientIndexView.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/ClientIndexView.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/ClientIndexViewRefactor.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/DataServiceTupleIterator.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/IClientIndex.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/IScaleOutClientIndex.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/PartitionedTupleIterator.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/RawDataServiceTupleIterator.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/SimpleDataServiceProcedureTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/pipeline/AbstractPendingSetSubtask.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/pipeline/IndexPartitionWriteTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/pipeline/IndexWriteTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/util/config/NicUtil.java
branches/dev-btm/bigdata/src/resources/logging/log4j.properties
branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestTransactionService.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/MockTransactionService.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestAddDeleteResource.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestBuildTask2.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/AbstractEmbeddedFederationTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/StressTestConcurrent.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestBasicIndexStuff.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestDistributedTransactionService.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestEmbeddedClient.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestEventParser.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestEventReceiver.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestMove.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestOverflow.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestRangeQuery.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestRangeQueryRemote.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestRestartSafe.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestScatterSplit.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestSplitJoin.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/boot/config/boot-processes.xml
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/BigdataZooDefs.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ManageLogicalServiceTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServicesManagerServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/BigdataServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/DataServerConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/JiniServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/MaxDataServicesPerHostConstraint.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServicesManagerConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/EmbeddedLoadBalancer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/Constants.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/EmbeddedShardLocator.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/PrivateInterface.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/ServiceProxy.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/logging.properties
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/ClientServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/DataServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/JiniFederation.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/MetadataServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/benchmark/ThroughputMaster.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/lookup/BigdataCachingServiceClient.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/lookup/DataServicesClient.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/AbstractAsynchronousClientTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/AbstractClientTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/AbstractResourceScanner.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/DiscoverServices.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/IAsynchronousClientTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/IResourceScannerFactory.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/MappedTaskMaster.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/ResourceBufferTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/ServiceMap.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/TaskMaster.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/DumpFederation.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/JiniServicesHelper.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/AdminProxy.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/Constants.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/PrivateInterface.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/ServiceProxy.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/config/logging.properties
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/config/shard.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/EmbeddedTransactionService.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/EntryUtil.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/Util.java
branches/dev-btm/bigdata-jini/src/resources/config/bigdataStandaloneTesting.config
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/AbstractServerTestCase.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/PerformanceTest.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/TestBigdataClient.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/master/TestMappedRDFDataLoadMaster.config
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFDataLoadMaster.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFFileLoadTask.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/RDFJoinNexus.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPORelation.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/AbstractServerTestCase.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/stress/LoadClosureAndQueryTest.java
branches/dev-btm/src/resources/bin/config/browser.config
branches/dev-btm/src/resources/config/bigdataCluster.config
Added Paths:
-----------
branches/dev-btm/bigdata/src/java/com/bigdata/counters/IDataServiceCounters.java
branches/dev-btm/bigdata/src/java/com/bigdata/counters/LoadBalancerReportingTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/counters/ReadBlockCounters.java
branches/dev-btm/bigdata/src/java/com/bigdata/counters/httpd/HttpReportingServer.java
branches/dev-btm/bigdata/src/java/com/bigdata/event/
branches/dev-btm/bigdata/src/java/com/bigdata/event/EventQueue.java
branches/dev-btm/bigdata/src/java/com/bigdata/event/EventQueueSenderTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/DistributedCommitTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/EmbeddedIndexStore.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/GetIndexMetadataTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/LocalTransactionManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/RangeIteratorTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/TxState.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/LocalResourceManagement.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/DataTaskWrapper.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IClientServiceCallable.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/MetadataIndexCachePolicy.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/OverflowAdmin.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/FileServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/EmbeddedShardService.java
Removed Paths:
-------------
branches/dev-btm/bigdata/src/java/com/bigdata/service/TxState.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/EmbeddedIndexStore.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/TestAdmin.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/TestAdmin.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/TestAdmin.java
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java 2010-10-04 13:27:07 UTC (rev 3721)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -44,7 +44,6 @@
import com.bigdata.relation.locator.DefaultResourceLocator;
import com.bigdata.search.FullTextIndex;
import com.bigdata.service.IBigdataFederation;
-import com.bigdata.service.IDataService;
import com.bigdata.sparse.AutoIncIntegerCounter;
import com.bigdata.sparse.IRowStoreConstants;
import com.bigdata.sparse.ITPS;
@@ -259,12 +258,12 @@
* <p>
* Note: The {@link BigdataFileSystem} makes the <strong>assumption</strong>
* that the {@link com.bigdata.journal.Options#OFFSET_BITS} is the #of
- * offset bits configured for the {@link IDataService}s in the connected
+ * offset bits configured for the shard services in the connected
* {@link IBigdataFederation} and computes the
* {@link BigdataFileSystem#getBlockSize()} based on that assumption. It is
* NOT possible to write blocks on the {@link BigdataFileSystem} whose size
* is greater than the maximum block size actually configured for the
- * {@link IDataService}s in the connected {@link IBigdataFederation}.
+ * shard services in the connected {@link IBigdataFederation}.
*
* @see com.bigdata.journal.Options#OFFSET_BITS
* @see #getOffsetBits()
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/IRangeQuery.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/IRangeQuery.java 2010-10-04 13:27:07 UTC (rev 3721)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/IRangeQuery.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -36,7 +36,6 @@
import com.bigdata.btree.filter.ITupleFilter;
import com.bigdata.btree.filter.TupleRemover;
import com.bigdata.service.IBigdataClient;
-import com.bigdata.service.IDataService;
import com.bigdata.service.ndx.IClientIndex;
import cutthecrap.utils.striterators.Striterator;
@@ -190,9 +189,9 @@
* <p>
* Note: This semantics of this flag require that the entries are atomically
* removed within the isolation level of the operation. In particular, if
- * the iterator is running against an {@link IDataService} using an
- * unisolated view then the entries MUST be buffered and removed as the
- * {@link ResultSet} is populated.
+ * the iterator is running against a shard service using an unisolated view
+ * then the entries MUST be buffered and removed as the {@link ResultSet}
+ * is populated.
* <p>
* Note: The {@link BigdataFileSystem#deleteHead(String, int)} relies on
* this atomic guarantee.
@@ -274,8 +273,7 @@
* processing which benefits from order effects can exploit the within chunk
* ordering of the tuples. The semantics of this flag are realized by the
* {@link IClientIndex} implementation. This flag has no effect on an
- * {@link ILocalBTreeView} and is not passed through to the
- * {@link IDataService}.
+ * {@link ILocalBTreeView} and is not passed through to the shard service.
*
* @todo This flag is not supported in combination with {@link #CURSOR}?
* @todo This flag is not supported in combination with {@link #REVERSE}?
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexMetadata.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexMetadata.java 2010-10-04 13:27:07 UTC (rev 3721)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexMetadata.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -65,12 +65,17 @@
import com.bigdata.resources.OverflowManager;
import com.bigdata.resources.StaleLocatorException;
import com.bigdata.service.AbstractFederation;
-import com.bigdata.service.DataService;
+//BTM import com.bigdata.service.DataService;
import com.bigdata.service.IBigdataFederation;
-import com.bigdata.service.IDataService;
+//BTM import com.bigdata.service.IDataService;
import com.bigdata.service.ndx.pipeline.AbstractSubtask;
import com.bigdata.sparse.SparseRowStore;
+//BTM
+import com.bigdata.service.IService;
+import com.bigdata.service.Service;
+import com.bigdata.service.ShardService;
+
/**
* <p>
* The persistent and mostly immutable metadata for a {@link AbstractBTree}.
@@ -904,7 +909,7 @@
* Boolean option indicates whether or not scatter splits are performed
* (default {@value #SCATTER_SPLIT_ENABLED}). Scatter splits only apply
* for scale-out indices where they "scatter" the initial index
- * partition across the {@link IDataService}s in the federation. This
+ * partition across the {@link ShardService}s in the federation. This
* is normally very useful.
* <P>
* Sometimes a scatter split is not the "right" thing for an index. An
@@ -1012,7 +1017,7 @@
}
/**
- * The {@link UUID} of the {@link DataService} on which the first partition
+ * The {@link UUID} of the {@link ShardService} on which the first partition
* of the scale-out index should be created. This is a purely transient
* property and will be <code>null</code> unless either explicitly set or
* set using {@value Options#INITIAL_DATA_SERVICE}. This property is only
@@ -1928,14 +1933,19 @@
final IBigdataFederation<?> fed = (IBigdataFederation<?>) indexManager;
- final IDataService dataService = fed
- .getDataServiceByName(val);
+//BTM final IDataService dataService = fed.getDataServiceByName(val);
+final ShardService dataService = fed.getDataServiceByName(val);
if (dataService != null) {
try {
- uuid = dataService.getServiceUUID();
+//BTM uuid = dataService.getServiceUUID();
+if(dataService instanceof IService) {
+ uuid = ((IService)dataService).getServiceUUID();
+} else {
+ uuid = ((Service)dataService).getServiceUUID();
+}
} catch (IOException ex) {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegment.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegment.java 2010-10-04 13:27:07 UTC (rev 3721)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegment.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -310,10 +310,15 @@
if (fileStore.fed != null) {
- openCloseEvent = new Event(fileStore.fed, new EventResource(
- fileStore.getIndexMetadata(), fileStore.file),
- EventType.IndexSegmentOpenClose);
-
+//BTM openCloseEvent = new Event(fileStore.fed, new EventResource(
+//BTM fileStore.getIndexMetadata(), fileStore.file),
+//BTM EventType.IndexSegmentOpenClose);
+openCloseEvent = new Event( (fileStore.fed).getEventQueue(),
+ (fileStore.fed).getServiceIface(),
+ (fileStore.fed).getServiceName(),
+ (fileStore.fed).getServiceUUID(),
+ new EventResource(fileStore.getIndexMetadata(), fileStore.file),
+ EventType.IndexSegmentOpenClose);
}
if (!fileStore.isOpen()) {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentStore.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentStore.java 2010-10-04 13:27:07 UTC (rev 3721)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentStore.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -447,8 +447,11 @@
if (fed != null) {
- openCloseEvent = new Event(fed, new EventResource(
- indexMetadata, file),
+//BTM openCloseEvent = new Event(fed, new EventResource(
+//BTM indexMetadata, file),
+//BTM EventType.IndexSegmentStoreOpenClose).start();
+openCloseEvent = new Event( fed.getEventQueue(), fed.getServiceIface(), fed.getServiceName(), fed.getServiceUUID(),
+ new EventResource(indexMetadata, file),
EventType.IndexSegmentStoreOpenClose).start();
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/ResultSet.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/ResultSet.java 2010-10-04 13:27:07 UTC (rev 3721)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/ResultSet.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -55,7 +55,6 @@
import com.bigdata.io.FixedByteArrayBuffer;
import com.bigdata.journal.ITx;
import com.bigdata.mdi.IResourceMetadata;
-import com.bigdata.service.IDataService;
/**
* An object used to stream key scan results back to the client.
@@ -283,7 +282,7 @@
* The values returned by {@link ITuple#getSourceIndex()} may be used to
* identify the resource from which a given tuple was read. That information
* is used to direct {@link ITuple#readBlock(long)} requests to the correct
- * resource on the {@link IDataService}.
+ * resource on the shard service.
*/
final public IResourceMetadata[] getSources() {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/proc/AbstractKeyArrayIndexProcedureConstructor.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/proc/AbstractKeyArrayIndexProcedureConstructor.java 2010-10-04 13:27:07 UTC (rev 3721)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/proc/AbstractKeyArrayIndexProcedureConstructor.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -5,14 +5,12 @@
import com.bigdata.btree.ITupleSerializer;
import com.bigdata.btree.IndexMetadata;
import com.bigdata.btree.raba.codec.IRabaCoder;
-import com.bigdata.service.IDataService;
/**
* A factory for {@link IKeyArrayIndexProcedure}s so that their data may be key
* range partitions and mapped against each relevant index partition.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public abstract class AbstractKeyArrayIndexProcedureConstructor<T extends IKeyArrayIndexProcedure> {
@@ -29,7 +27,8 @@
*
* @param ndx
* The index - this is used to determine the serializers for the
- * keys and/or values to be sent to a remote {@link IDataService}.
+ * keys and/or values to be sent to the backend of the shard
+ * service.
* @param fromIndex
* The index of the first key to be used (inclusive).
* @param toIndex
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/proc/IIndexProcedure.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/proc/IIndexProcedure.java 2010-10-04 13:27:07 UTC (rev 3721)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/proc/IIndexProcedure.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -34,8 +34,6 @@
import com.bigdata.btree.IIndex;
import com.bigdata.btree.IRangeQuery;
import com.bigdata.btree.ISimpleBTree;
-import com.bigdata.service.DataService;
-import com.bigdata.service.IDataService;
import com.bigdata.service.ndx.ClientIndexView;
import com.bigdata.sparse.SparseRowStore;
@@ -86,15 +84,15 @@
* </dl>
*
* Note: this interface extends {@link Serializable}, however that provides
- * only for communicating state to the {@link IDataService}. If an instance of
+ * only for communicating state to the shard service. If an instance of
* this procedure will cross a network interface, then the implementation class
- * MUST be available to the {@link IDataService} on which it will execute. This
+ * MUST be available to the shard service on which it will execute. This
* can be as simple as bundling the procedure into a JAR that is part of the
- * CLASSPATH used to start a {@link DataService} or you can use downloaded code
- * with the JINI codebase mechanism (<code>java.rmi.server.codebase</code>).
+ * CLASSPATH used to start the frontend data service or backend embedded
+ * shard service, or you can use downloaded code with the JINI codebase
+ * mechanism (<code>java.rmi.server.codebase</code>).
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*
* @todo add generic type for {@link #apply(IIndex)} 's return value (much like
* {@link Callable}).
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/config/Configuration.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/config/Configuration.java 2010-10-04 13:27:07 UTC (rev 3721)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/config/Configuration.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -38,11 +38,16 @@
import com.bigdata.btree.IndexMetadata;
import com.bigdata.journal.IIndexManager;
import com.bigdata.relation.RelationSchema;
-import com.bigdata.service.DataService;
+//BTM import com.bigdata.service.DataService;
import com.bigdata.service.IBigdataFederation;
-import com.bigdata.service.IDataService;
+//BTM import com.bigdata.service.IDataService;
import com.bigdata.util.NV;
+//BTM
+import com.bigdata.service.IService;
+import com.bigdata.service.Service;
+import com.bigdata.service.ShardService;
+
/**
* Base class for managing the initial configuration metadata for indices and
* locatable resources.
@@ -317,7 +322,7 @@
// }
/**
- * Resolve the value to a {@link DataService} {@link UUID}.
+ * Resolve the value to a {@link ShardService} {@link UUID}.
*
* @param indexManager
* The index manager (optional).
@@ -365,14 +370,19 @@
*/
{
- final IDataService dataService = fed.getDataServiceByName(val);
+//BTM final IDataService dataService = fed.getDataServiceByName(val);
+final ShardService dataService = fed.getDataServiceByName(val);
if (dataService != null) {
try {
- return dataService.getServiceUUID();
-
+//BTM return dataService.getServiceUUID();
+if(dataService instanceof IService) {
+ return ((IService)dataService).getServiceUUID();
+} else {
+ return ((Service)dataService).getServiceUUID();
+}
} catch (IOException ex) {
throw new RuntimeException(ex);
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/counters/CounterSet.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/counters/CounterSet.java 2010-10-04 13:27:07 UTC (rev 3721)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/counters/CounterSet.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -785,6 +785,8 @@
// counter exists for that path.
log.error("Exists: path=" + getPath() + ", name=" + name);
+//BTM
+System.out.println("\n"+com.bigdata.util.Util.getCurrentStackTrace());
// return existing counter for path @todo vs replace.
return (ICounter)counter;
Added: branches/dev-btm/bigdata/src/java/com/bigdata/counters/IDataServiceCounters.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/counters/IDataServiceCounters.java (rev 0)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/counters/IDataServiceCounters.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -0,0 +1,53 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package com.bigdata.counters;
+
+import com.bigdata.journal.ConcurrencyManager;
+import com.bigdata.resources.ResourceManager;
+
+/**
+ * Interface that defines and documents the counters and counter namespaces
+ * reported by the shard (data) service and the various services which it
+ * uses.
+ */
+public interface IDataServiceCounters
+ extends ConcurrencyManager.IConcurrencyManagerCounters,
+ ResourceManager.IResourceManagerCounters
+{
+ /**
+ * The namespace for the counters pertaining to the {@link ConcurrencyManager}.
+ */
+ String concurrencyManager = "Concurrency Manager";
+
+ /**
+ * The namespace for the counters pertaining to the {@link ILocalTransactionService}.
+ */
+ String transactionManager = "Transaction Manager";
+
+ /**
+ * The namespace for the counters pertaining to the {@link ResourceManager}.
+ */
+ String resourceManager = "Resource Manager";
+}
\ No newline at end of file
Property changes on: branches/dev-btm/bigdata/src/java/com/bigdata/counters/IDataServiceCounters.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/dev-btm/bigdata/src/java/com/bigdata/counters/LoadBalancerReportingTask.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/counters/LoadBalancerReportingTask.java (rev 0)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/counters/LoadBalancerReportingTask.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -0,0 +1,124 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package com.bigdata.counters;
+
+import com.bigdata.counters.CounterSet;
+import com.bigdata.rawstore.Bytes;
+import com.bigdata.service.IFederationDelegate;
+import com.bigdata.service.LoadBalancer;
+import com.bigdata.util.config.LogUtil;
+
+import net.jini.core.lookup.ServiceItem;
+import net.jini.lookup.LookupCache;
+
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.UUID;
+
+/**
+ * <code>Runnable</code> task class that periodically reports
+ * performance counter data to the load balancer service.
+ * (Note: see AbstractFederation.ReportTask).
+ */
+public class LoadBalancerReportingTask implements Runnable {
+
+ private IFederationDelegate embeddedIndexStore;
+ private UUID serviceUUID;
+ private CounterSet serviceRoot;
+ private LookupCache lbsCache;//for discovering lbs
+ private LoadBalancer embeddedLbs;//for testing embedded fed
+ private Logger logger;
+
+ public LoadBalancerReportingTask
+ (IFederationDelegate embeddedIndexStore,
+ UUID serviceUUID,
+ CounterSet serviceRoot,
+ LookupCache loadBalancerCache,
+ LoadBalancer embeddedLoadBalancer,
+ Logger logger)
+ {
+ this.embeddedIndexStore = embeddedIndexStore;
+ this.serviceUUID = serviceUUID;
+ this.serviceRoot = serviceRoot;
+ this.lbsCache = loadBalancerCache;
+ this.embeddedLbs = embeddedLoadBalancer;//for embedded fed testing
+ this.logger = (logger == null ?
+ LogUtil.getLog4jLogger((this.getClass()).getName()) :
+ logger);
+ }
+
+ public void run() {
+ try {
+ embeddedIndexStore.reattachDynamicCounters();
+ } catch (Throwable t) {
+ logger.error
+ ("failure on dynamic counter reattachment ["+t+"]", t);
+ }
+ try {
+ // Report the performance counters to the load balancer.
+ reportPerformanceCounters();
+ } catch (Throwable t) {
+ logger.error("failure while reporting performance "
+ +"counters to load balancer ["+t+"]", t);
+ }
+ }
+
+ private void reportPerformanceCounters() throws IOException {
+System.out.println("\n>>>>> LoadBalancerReportingTask.reportPerformanceCounters: serviceUUID = "+serviceUUID);
+ LoadBalancer lbs = null;
+ if(embeddedLbs != null) {
+ lbs = embeddedLbs;
+ } else {
+ if(lbsCache != null) {
+ ServiceItem lbsItem = lbsCache.lookup(null);
+ if(lbsItem != null) {
+ lbs = (LoadBalancer)(lbsItem.service);
+ }
+ }
+ }
+ if(lbs == null) {
+ logger.warn
+ ("cannot report counters [no load balancer service]");
+System.out.println(">>>>> LoadBalancerReportingTask.reportPerformanceCounters: loadBalancerService = NULL");
+ return;
+ }
+
+System.out.println(">>>>> LoadBalancerReportingTask.reportPerformanceCounters: loadBalancerService = "+lbs);
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream(Bytes.kilobyte32 * 2);
+ serviceRoot.asXML(baos, "UTF-8", null/* filter */);
+
+System.out.println(">>>>> LoadBalancerReportingTask.reportPerformanceCounters: CALLING loadBalancer.notify ...");
+ lbs.notify(serviceUUID, baos.toByteArray());
+System.out.println(">>>>> LoadBalancerReportingTask.reportPerformanceCounters: DONE CALLING loadBalancer.notify");
+
+ if (logger.isDebugEnabled()) {
+ logger.debug("report counters sent to load balancer");
+ }
+ }
+}
Property changes on: branches/dev-btm/bigdata/src/java/com/bigdata/counters/LoadBalancerReportingTask.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/dev-btm/bigdata/src/java/com/bigdata/counters/ReadBlockCounters.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/counters/ReadBlockCounters.java (rev 0)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/counters/ReadBlockCounters.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -0,0 +1,45 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package com.bigdata.counters;
+
+/**
+ * Class that can be used as a data structure that encapsulates
+ * the current state of the read block counters maintained by
+ * the shard (data) service.
+ *
+ * @todo improve reporting here and for block write as well
+ * (goes through unisolated tasks at the present).
+ */
+public class ReadBlockCounters {
+
+ /** #of block read requests. */
+ public long readBlockCount;
+ public long readBlockErrorCount;
+ public long readBlockBytes;
+ public long readBlockNanos;
+
+ public ReadBlockCounters() { }
+}
+
Property changes on: branches/dev-btm/bigdata/src/java/com/bigdata/counters/ReadBlockCounters.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/dev-btm/bigdata/src/java/com/bigdata/counters/httpd/HttpReportingServer.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/counters/httpd/HttpReportingServer.java (rev 0)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/counters/httpd/HttpReportingServer.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -0,0 +1,79 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package com.bigdata.counters.httpd;
+
+import com.bigdata.counters.CounterSet;
+import com.bigdata.service.IFederationDelegate;
+import com.bigdata.util.config.LogUtil;
+
+import org.apache.log4j.Logger;
+
+import java.io.IOException;
+import java.util.Properties;
+import java.util.LinkedHashMap;
+import java.util.Vector;
+
+/**
+ * Overrides the method <code>doGet</code> in
+ * <code>CounterSetHTTPD</code>.
+ */
+public class HttpReportingServer extends CounterSetHTTPD {
+
+ private IFederationDelegate embeddedIndexStore;
+ private Logger logger;
+
+ public HttpReportingServer
+ (final int port,
+ final CounterSet root,
+ final IFederationDelegate embeddedIndexStore,
+ Logger logger)
+ throws IOException
+ {
+ super(port, root);
+ this.embeddedIndexStore = embeddedIndexStore;
+ this.logger = (logger == null ?
+ LogUtil.getLog4jLogger((this.getClass()).getName()) :
+ logger);
+ }
+
+ @Override
+ public Response doGet(String uri,
+ String method,
+ Properties header,
+ LinkedHashMap<String,
+ Vector<String>> parms)
+ throws Exception
+ {
+ try {
+ embeddedIndexStore.reattachDynamicCounters();
+ } catch (Exception e) {
+ // Usually because the live journal has been
+ // concurrently closed during the request.
+ logger.error
+ ("failure on dynamic counter reattachment ["+e+"]", e);
+ }
+ return super.doGet(uri, method, header, parms);
+ }
+}
Property changes on: branches/dev-btm/bigdata/src/java/com/bigdata/counters/httpd/HttpReportingServer.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/dev-btm/bigdata/src/java/com/bigdata/event/EventQueue.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/event/EventQueue.java (rev 0)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/event/EventQueue.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -0,0 +1,33 @@
+/*
+
+Copyright (C) SYSTAP, LLC 2006-2008. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+*/
+
+package com.bigdata.event;
+
+import com.bigdata.service.Event;
+
+public interface EventQueue {
+
+ void queueEvent(Event e);
+}
Property changes on: branches/dev-btm/bigdata/src/java/com/bigdata/event/EventQueue.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/dev-btm/bigdata/src/java/com/bigdata/event/EventQueueSenderTask.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/event/EventQueueSenderTask.java (rev 0)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/event/EventQueueSenderTask.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -0,0 +1,118 @@
+/*
+
+Copyright (C) SYSTAP, LLC 2006-2008. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+*/
+
+package com.bigdata.event;
+
+import com.bigdata.service.Event;
+import com.bigdata.service.EventReceivingService;
+import com.bigdata.service.LoadBalancer;
+import com.bigdata.util.config.LogUtil;
+
+import net.jini.core.lookup.ServiceItem;
+import net.jini.lookup.LookupCache;
+
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+import java.util.LinkedList;
+import java.util.concurrent.BlockingQueue;
+
+public class EventQueueSenderTask implements EventQueue, Runnable {
+
+ private BlockingQueue<Event> eventQueue;
+ private LookupCache lbsCache;//for discovering lbs
+ private LoadBalancer embeddedLbs;//for testing embedded fed
+ private String serviceName;
+ private Logger logger;
+
+ public EventQueueSenderTask(BlockingQueue<Event> eventQueue,
+ LookupCache loadBalancerCache,
+ LoadBalancer embeddedLoadBalancer,
+ String serviceName,
+ Logger logger)
+ {
+ this.eventQueue = eventQueue;
+ this.lbsCache = loadBalancerCache;
+ this.embeddedLbs = embeddedLoadBalancer;//for embedded fed testing
+
+ this.serviceName = serviceName;//for debug output
+ this.logger = (logger == null ?
+ LogUtil.getLog4jLogger((this.getClass()).getName()) :
+ logger);
+ }
+
+ // Required by EventQueue interface
+
+ public void queueEvent(Event e) {
+ eventQueue.add(e);
+ }
+
+ // Required by Runnable interface
+
+ public void run() {
+
+//BTM - for now, maintain the same logic and functionality as that in
+//BTM the class AbstractFederation#SendEventsTask
+
+ try {
+ LoadBalancer lbs = null;
+ EventReceivingService serviceRef = null;
+ if(embeddedLbs != null) {
+ lbs = embeddedLbs;
+ } else {
+ if(lbsCache != null) {
+ ServiceItem lbsItem = lbsCache.lookup(null);
+ if(lbsItem != null) {
+ lbs = (LoadBalancer)(lbsItem.service);
+ }
+ }
+ }
+ if(lbs == null) return;
+ serviceRef = (EventReceivingService)lbs;
+
+ final long begin = System.currentTimeMillis();//for logging
+
+ final LinkedList<Event> queuedEvents = new LinkedList<Event>();
+ eventQueue.drainTo(queuedEvents);
+
+ for (Event event : queuedEvents) {
+ serviceRef.notifyEvent(event);//don't sync on remote call
+ }
+
+ if (logger.isDebugEnabled()) {
+ final int nevents = queuedEvents.size();
+ if (nevents > 0) {
+ logger.log
+ (Level.DEBUG, nevents+" event(s) sent to load "
+ +"balancer ["+(System.currentTimeMillis()-begin)
+ +" ms]");
+ }
+ }
+ } catch (Throwable t) {
+ logger.log(Level.WARN, "while sending events to load "
+ +"balancer from "+serviceName, t);
+ }
+ }
+}
Property changes on: branches/dev-btm/bigdata/src/java/com/bigdata/event/EventQueueSenderTask.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java 2010-10-04 13:27:07 UTC (rev 3721)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -9,15 +9,13 @@
import com.bigdata.counters.Instrument;
import com.bigdata.resources.StoreManager;
import com.bigdata.service.IBigdataFederation;
-import com.bigdata.service.IDataService;
/**
* Manages the client side of a transaction either for a standalone
- * {@link Journal} or for an {@link IDataService} in an
+ * {@link Journal} or for a shard service in an
* {@link IBigdataFederation}.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
abstract public class AbstractLocalTransactionManager implements
ILocalTransactionManager {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/ConcurrencyManager.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/ConcurrencyManager.java 2010-10-04 13:27:07 UTC (rev 3721)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/ConcurrencyManager.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -86,27 +86,25 @@
* </dd>
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public class ConcurrencyManager implements IConcurrencyManager {
- final protected static Logger log = Logger.getLogger(ConcurrencyManager.class);
+ final private static Logger log = Logger.getLogger(ConcurrencyManager.class);
// /**
// * True iff the {@link #log} level is INFO or less.
// */
-// final protected static boolean INFO = log.isInfoEnabled();
+// final private static boolean INFO = log.isInfoEnabled();
/**
* True iff the {@link #log} level is DEBUG or less.
*/
- final protected static boolean DEBUG = log.isDebugEnabled();
+ final private static boolean DEBUG = log.isDebugEnabled();
/**
* Options for the {@link ConcurrentManager}.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public static interface Options extends IServiceShutdown.Options {
@@ -340,7 +338,7 @@
* Once the transaction has acquired those writable indices it then runs its
* commit phrase as an unisolated operation on the {@link #writeService}.
*/
- final protected ThreadPoolExecutor txWriteService;
+ final private ThreadPoolExecutor txWriteService;
/**
* Pool of threads for handling concurrent unisolated read operations on
@@ -358,7 +356,7 @@
* historical commit records (which may span more than one logical
* journal) until the reader terminates.
*/
- final protected ThreadPoolExecutor readService;
+ final private ThreadPoolExecutor readService;
/**
* Pool of threads for handling concurrent unisolated write operations on
@@ -371,6 +369,7 @@
* Serialization of access to unisolated named indices is acomplished by
* gaining an exclusive lock on the unisolated named index.
*/
+ // protected for access by tests.
final protected WriteExecutorService writeService;
/**
@@ -400,7 +399,7 @@
}
- protected void assertOpen() {
+ private void assertOpen() {
if (!open)
throw new IllegalStateException();
@@ -466,7 +465,7 @@
* Long.MAX_VALUE.
*/
- final long shutdownTimeout = this.shutdownTimeout == 0L ? Long.MAX_VALUE
+ final long tmpShutdownTimeout = this.shutdownTimeout == 0L ? Long.MAX_VALUE
: this.shutdownTimeout;
final TimeUnit unit = TimeUnit.MILLISECONDS;
@@ -486,7 +485,7 @@
final long elapsed = System.currentTimeMillis() - begin;
- if(!txWriteService.awaitTermination(shutdownTimeout-elapsed, unit)) {
+ if(!txWriteService.awaitTermination(tmpShutdownTimeout-elapsed, unit)) {
log.warn("Transaction service termination: timeout");
@@ -505,7 +504,7 @@
final long elapsed = System.currentTimeMillis() - begin;
- if(!readService.awaitTermination(shutdownTimeout-elapsed, unit)) {
+ if(!readService.awaitTermination(tmpShutdownTimeout-elapsed, unit)) {
log.warn("Read service termination: timeout");
@@ -521,7 +520,7 @@
final long elapsed = System.currentTimeMillis() - begin;
- final long timeout = shutdownTimeout-elapsed;
+ final long timeout = tmpShutdownTimeout-elapsed;
if (log.isInfoEnabled())
log.info("Awaiting write service termination: will wait "
@@ -921,13 +920,13 @@
}
/** Counters for {@link #writeService}. */
- protected final WriteTaskCounters countersUN = new WriteTaskCounters();
+ final WriteTaskCounters countersUN = new WriteTaskCounters();
/** Counters for the {@link #txWriteService}. */
- protected final TaskCounters countersTX = new TaskCounters();
+ final TaskCounters countersTX = new TaskCounters();
/** Counters for the {@link #readService}. */
- protected final TaskCounters countersHR = new TaskCounters();
+ final TaskCounters countersHR = new TaskCounters();
/**
* Sampling instruments for the various queues giving us the moving average
@@ -942,7 +941,6 @@
* the {@link ConcurrencyManager}.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public static interface IConcurrencyManagerCounters {
@@ -977,7 +975,6 @@
* Reports the elapsed time since the service was started.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
private static class ServiceElapsedTimeInstrument extends Instrument<Long> {
Added: branches/dev-btm/bigdata/src/java/com/bigdata/journal/DistributedCommitTask.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/DistributedCommitTask.java (rev 0)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/DistributedCommitTask.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -0,0 +1,211 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have rece...
[truncated message content] |
|
From: <btm...@us...> - 2010-10-04 23:12:53
|
Revision: 3728
http://bigdata.svn.sourceforge.net/bigdata/?rev=3728&view=rev
Author: btmurphy
Date: 2010-10-04 23:12:46 +0000 (Mon, 04 Oct 2010)
Log Message:
-----------
[branch dev-btm]: hand-merged changeset 3542 from trunk --> dev-btm
Modified Paths:
--------------
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFDataLoadMaster.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFFileLoadTask.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/SingleResourceReaderTask.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/BasicRioLoader.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/IRioLoader.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/PresortRioLoader.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util/Splitter.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/AbstractRIOTestCase.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/EDSAsyncLoader.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/TestAsynchronousStatementBufferFactory.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/stress/LoadClosureAndQueryTest.java
Modified: branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFDataLoadMaster.java
===================================================================
--- branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFDataLoadMaster.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFDataLoadMaster.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -272,6 +272,17 @@
// /** {@value #DEFAULT_MAX_TRIES} */
// int DEFAULT_MAX_TRIES = 3;
+ /**
+ * The value that will be used for the graph/context co-ordinate when
+ * loading data represented in a triple format into a quad store.
+ */
+ String DEFAULT_GRAPH = "defaultGraph" ;
+
+ /**
+ * TODO Should we always enforce a real value? i.e. provide a real default
+ * or abort the load.
+ */
+ String DEFAULT_DEFAULT_GRAPH = null ;
}
/**
@@ -400,6 +411,12 @@
private transient RDFFormat rdfFormat;
/**
+ * The value that will be used for the graph/context co-ordinate when
+ * loading data represented in a triple format into a quad store.
+ */
+ public final String defaultGraph ;
+
+ /**
* Force the load of the NxParser integration class and its registration
* of the NQuadsParser#nquads RDFFormat.
*
@@ -494,6 +511,8 @@
sb.append(", " + ConfigurationOptions.RDF_FORMAT + "=" + rdfFormat);
+ sb.append(", " + ConfigurationOptions.DEFAULT_GRAPH + "=" + defaultGraph) ;
+
sb.append(", " + ConfigurationOptions.FORCE_OVERFLOW_BEFORE_CLOSURE + "="
+ forceOverflowBeforeClosure);
@@ -624,6 +643,10 @@
}
+ defaultGraph = (String) config.getEntry(component,
+ ConfigurationOptions.DEFAULT_GRAPH, String.class,
+ ConfigurationOptions.DEFAULT_DEFAULT_GRAPH);
+
rejectedExecutionDelay = (Long) config.getEntry(
component,
ConfigurationOptions.REJECTED_EXECUTION_DELAY, Long.TYPE,
@@ -1005,6 +1028,7 @@
jobState.ontology,//file
jobState.ontology.getPath(),//baseURI
jobState.getRDFFormat(),//
+ jobState.defaultGraph,
jobState.ontologyFileFilter //
);
Modified: branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFFileLoadTask.java
===================================================================
--- branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFFileLoadTask.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFFileLoadTask.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -213,7 +213,8 @@
jobState.producerChunkSize,//
jobState.valuesInitialCapacity,//
jobState.bnodesInitialCapacity,//
- jobState.getRDFFormat(), //
+ jobState.getRDFFormat(), //
+ jobState.defaultGraph,
parserOptions,//
false, // deleteAfter is handled by the master!
jobState.parserPoolSize, //
Modified: branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/SingleResourceReaderTask.java
===================================================================
--- branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/SingleResourceReaderTask.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/SingleResourceReaderTask.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -23,7 +23,6 @@
* load and data verify is just the behavior of the {@link IStatementBuffer}.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public class SingleResourceReaderTask implements Runnable {
@@ -186,7 +185,7 @@
// run the parser.
// @todo reuse the same underlying parser instance?
- loader.loadRdf(reader, baseURL, rdfFormat, parserOptions);
+ loader.loadRdf(reader, baseURL, rdfFormat, null, parserOptions);
success = true;
Modified: branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java
===================================================================
--- branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -177,7 +177,6 @@
* Note: This DOES NOT support SIDS.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*
* FIXME Modify to support SIDs. We basically need to loop in the
* {@link #workflowLatch_bufferTerm2Id} workflow state until all SIDs have been
@@ -305,7 +304,6 @@
* write API for BTree and friends).
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*
* @param <S>
* The generic type of the statement objects.
@@ -356,8 +354,14 @@
* The default {@link RDFFormat}.
*/
private final RDFFormat defaultFormat;
-
+
/**
+ * The value that will be used for the graph/context co-ordinate when
+ * loading data represented in a triple format into a quad store.
+ */
+ private final String defaultGraph;
+
+ /**
* Options for the {@link RDFParser}.
*/
private final RDFParserOptions parserOptions;
@@ -1363,7 +1367,6 @@
* load and data verify is just the behavior of the {@link IStatementBuffer}.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
protected class ParserTask implements Callable<Void> {
@@ -1423,7 +1426,7 @@
try {
// run the parser.
new PresortRioLoader(buffer).loadRdf(reader, baseURL,
- rdfFormat, parserOptions);
+ rdfFormat, defaultGraph, parserOptions);
} finally {
reader.close();
}
@@ -1490,6 +1493,9 @@
* {@link BNode}s parsed from a single document.
* @param defaultFormat
* The default {@link RDFFormat} which will be assumed.
+ * @param defaultGraph
+ * The value that will be used for the graph/context co-ordinate when
+ * loading data represented in a triple format into a quad store.
* @param parserOptions
* Options for the {@link RDFParser}.
* @param deleteAfter
@@ -1529,6 +1535,7 @@
final int valuesInitialCapacity,//
final int bnodesInitialCapacity, //
final RDFFormat defaultFormat,//
+ final String defaultGraph,//
final RDFParserOptions parserOptions,//
final boolean deleteAfter,//
final int parserPoolSize,//
@@ -1566,6 +1573,8 @@
this.defaultFormat = defaultFormat;
+ this.defaultGraph = defaultGraph;
+
this.parserOptions = parserOptions;
this.deleteAfter = deleteAfter;
@@ -2250,7 +2259,6 @@
* Task deletes a resource from the local file system.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
protected class DeleteTask implements Runnable {
@@ -2712,7 +2720,6 @@
* the array is given by {@link Split#fromIndex}.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
static private class Term2IdWriteProcAsyncResultHandler
implements
@@ -2859,7 +2866,6 @@
*
* @author <a href="mailto:tho...@us...">Bryan
* Thompson</a>
- * @version $Id$
*
* @todo something similar for the SIDs
*/
@@ -2990,7 +2996,6 @@
*
* @author <a href="mailto:tho...@us...">Bryan
* Thompson</a>
- * @version $Id$
*/
static class AsyncId2TermIndexWriteTask implements Callable<Void> {
@@ -3140,7 +3145,6 @@
*
* @author <a href="mailto:tho...@us...">Bryan
* Thompson</a>
- * @version $Id$
*/
static class AsyncSPOIndexWriteTask implements Callable<Void> {
@@ -3277,7 +3281,6 @@
*
* @author <a href="mailto:tho...@us...">Bryan
* Thompson</a>
- * @version $Id$
* @param <S>
* @param <F>
*/
@@ -4083,7 +4086,6 @@
*
* @author <a href="mailto:tho...@us...">Bryan
* Thompson</a>
- * @version $Id$
*/
private class BufferOtherWritesTask implements Callable<Void> {
@@ -4153,7 +4155,6 @@
*
* @author <a href="mailto:tho...@us...">Bryan
* Thompson</a>
- * @version $Id$
*/
private class ParserThreadPoolExecutor extends ThreadPoolExecutor {
Modified: branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/BasicRioLoader.java
===================================================================
--- branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/BasicRioLoader.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/BasicRioLoader.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -41,7 +41,6 @@
* Parses data but does not load it into the indices.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public class BasicRioLoader implements IRioLoader {
@@ -74,6 +73,8 @@
private final ValueFactory valueFactory;
+ protected String defaultGraph;
+
public BasicRioLoader(final ValueFactory valueFactory) {
if (valueFactory == null)
@@ -153,18 +154,20 @@
}
final public void loadRdf(final InputStream is, final String baseURI,
- final RDFFormat rdfFormat, final RDFParserOptions options)
+ final RDFFormat rdfFormat, final String defaultGraph,
+ final RDFParserOptions options)
throws Exception {
- loadRdf2(is, baseURI, rdfFormat, options);
+ loadRdf2(is, baseURI, rdfFormat, defaultGraph, options);
}
final public void loadRdf(final Reader reader, final String baseURI,
- final RDFFormat rdfFormat, final RDFParserOptions options)
+ final RDFFormat rdfFormat, final String defaultGraph,
+ final RDFParserOptions options)
throws Exception {
- loadRdf2(reader, baseURI, rdfFormat, options);
+ loadRdf2(reader, baseURI, rdfFormat, defaultGraph, options);
}
@@ -180,7 +183,7 @@
* @throws Exception
*/
protected void loadRdf2(final Object source, final String baseURI,
- final RDFFormat rdfFormat, final RDFParserOptions options)
+ final RDFFormat rdfFormat, final String defaultGraph, final RDFParserOptions options)
throws Exception {
if (source == null)
@@ -198,6 +201,8 @@
if (log.isInfoEnabled())
log.info("format=" + rdfFormat + ", options=" + options);
+ this.defaultGraph = defaultGraph ;
+
final RDFParser parser = getParser(rdfFormat);
// apply options to the parser
@@ -212,7 +217,7 @@
// Note: reset so that rates are correct for each source loaded.
stmtsAdded = 0;
-
+
try {
before();
Modified: branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/IRioLoader.java
===================================================================
--- branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/IRioLoader.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/IRioLoader.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -37,7 +37,6 @@
* Interface for parsing RDF data using the Sesame RIO parser.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public interface IRioLoader {
@@ -72,12 +71,14 @@
* The base URL for those data.
* @param rdfFormat
* The interchange format.
+ * @param defaultGraph
+ * The default graph.
* @param options
* Options to be applied to the {@link RDFParser}.
* @throws Exception
*/
public void loadRdf(Reader reader, String baseURL, RDFFormat rdfFormat,
- RDFParserOptions options) throws Exception;
+ String defaultGraph, RDFParserOptions options) throws Exception;
/**
* Parse RDF data.
@@ -88,11 +89,13 @@
* The base URL for those data.
* @param rdfFormat
* The interchange format.
+ * @param defaultGraph
+ * The default graph.
* @param options
* Options to be applied to the {@link RDFParser}.
* @throws Exception
*/
public void loadRdf(InputStream is, String baseURI, RDFFormat rdfFormat,
- RDFParserOptions options) throws Exception;
+ String defaultGraph, RDFParserOptions options) throws Exception;
}
Modified: branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/PresortRioLoader.java
===================================================================
--- branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/PresortRioLoader.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/PresortRioLoader.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -23,17 +23,19 @@
*/
package com.bigdata.rdf.rio;
+import org.openrdf.model.Resource;
import org.openrdf.model.Statement;
import org.openrdf.model.Value;
import org.openrdf.rio.RDFHandler;
import org.openrdf.rio.RDFHandlerException;
+import com.bigdata.rdf.model.BigdataURI;
+
/**
* Statement handler for the RIO RDF Parser that writes on a
* {@link StatementBuffer}.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public class PresortRioLoader extends BasicRioLoader implements RDFHandler
{
@@ -45,6 +47,12 @@
final protected IStatementBuffer<?> buffer;
/**
+ * The value that will be used for the graph/context co-ordinate when
+ * loading data represented in a triple format into a quad store.
+ */
+ private BigdataURI defaultGraphURI = null ;
+
+ /**
* Sets up parser to load RDF.
*
* @param buffer
@@ -58,7 +66,7 @@
this.buffer = buffer;
}
-
+
/**
* bulk insert the buffered data into the store.
*/
@@ -87,8 +95,11 @@
public RDFHandler newRDFHandler() {
+ defaultGraphURI = null != defaultGraph && 4 == buffer.getDatabase ().getSPOKeyArity ()
+ ? buffer.getDatabase ().getValueFactory ().createURI ( defaultGraph )
+ : null
+ ;
return this;
-
}
public void handleStatement( final Statement stmt ) {
@@ -98,9 +109,13 @@
log.debug(stmt);
}
-
+
+ Resource graph = stmt.getContext() ;
+ if ( null == graph
+ && null != defaultGraphURI ) // only true when we know we are loading a quad store
+ graph = defaultGraphURI ;
// buffer the write (handles overflow).
- buffer.add( stmt.getSubject(), stmt.getPredicate(), stmt.getObject(), stmt.getContext() );
+ buffer.add( stmt.getSubject(), stmt.getPredicate(), stmt.getObject(), graph );
stmtsAdded++;
Modified: branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java
===================================================================
--- branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -638,7 +638,7 @@
final LoadStats totals = new LoadStats();
- loadData3(totals, reader, baseURL, rdfFormat, true/*endOfBatch*/);
+ loadData3(totals, reader, baseURL, rdfFormat, null, true/*endOfBatch*/);
return totals;
@@ -666,7 +666,7 @@
final LoadStats totals = new LoadStats();
- loadData3(totals, is, baseURL, rdfFormat, true/* endOfBatch */);
+ loadData3(totals, is, baseURL, rdfFormat, null, true/* endOfBatch */);
return totals;
@@ -702,7 +702,7 @@
final LoadStats totals = new LoadStats();
- loadData3(totals, is, baseURL, rdfFormat, true/*endOfBatch*/);
+ loadData3(totals, is, baseURL, rdfFormat, null, true/*endOfBatch*/);
return totals;
@@ -773,7 +773,7 @@
final File file = new File(resource);
if(file.exists()) {
loadFiles(totals, 0/* depth */, file.toURI().toURL(),
- baseURL, rdfFormat, filter, endOfBatch);
+ baseURL, rdfFormat, null, filter, endOfBatch);
return;
}
// not on the file system, try the classpath next
@@ -807,7 +807,7 @@
try {
- loadData3(totals, reader, baseURL, rdfFormat, endOfBatch);
+ loadData3(totals, reader, baseURL, rdfFormat, null, endOfBatch);
} catch (Exception ex) {
@@ -833,6 +833,9 @@
* The format of the file (optional, when not specified the
* format is deduced for each file in turn using the
* {@link RDFFormat} static methods).
+ * @param defaultGraph
+ * The value that will be used for the graph/context co-ordinate when
+ * loading data represented in a triple format into a quad store.
* @param filter
* A filter selecting the file names that will be loaded
* (optional). When specified, the filter MUST accept directories
@@ -843,7 +846,8 @@
* @throws IOException
*/
public LoadStats loadFiles(final URL url, final String baseURI,
- final RDFFormat rdfFormat, final FilenameFilter filter)
+ final RDFFormat rdfFormat, final String defaultGraph,
+ final FilenameFilter filter)
throws IOException {
if (url == null)
@@ -851,8 +855,8 @@
final LoadStats totals = new LoadStats();
- loadFiles(totals, 0/* depth */, url, baseURI,
- rdfFormat, filter, true/* endOfBatch */);
+ loadFiles(totals, 0/* depth */, url, baseURI, rdfFormat, defaultGraph, filter, true/* endOfBatch */
+ );
return totals;
@@ -860,7 +864,8 @@
protected void loadFiles(final LoadStats totals, final int depth,
final URL url, final String baseURI, final RDFFormat rdfFormat,
- final FilenameFilter filter, final boolean endOfBatch)
+ final String defaultGraph, final FilenameFilter filter,
+ final boolean endOfBatch)
throws IOException {
// Legacy behavior - allow local files and directories for now,
@@ -886,7 +891,7 @@
final File f = files[i];
loadFiles(totals, depth + 1, f.toURI().toURL(), baseURI,
- rdfFormat, filter,
+ rdfFormat, defaultGraph, filter,
(depth == 0 && i < files.length ? false : endOfBatch));
}
@@ -930,7 +935,7 @@
final String s = baseURI != null ? baseURI : url.toURI()
.toString();
- loadData3(totals, reader, s, rdfFormat, endOfBatch);
+ loadData3(totals, reader, s, rdfFormat, defaultGraph, endOfBatch);
return;
@@ -962,7 +967,7 @@
*/
protected void loadData3(final LoadStats totals, final Object source,
final String baseURL, final RDFFormat rdfFormat,
- final boolean endOfBatch) throws IOException {
+ final String defaultGraph, final boolean endOfBatch) throws IOException {
final long begin = System.currentTimeMillis();
@@ -985,11 +990,10 @@
}
// Setup the loader.
- final PresortRioLoader loader = new PresortRioLoader(buffer);
+ final PresortRioLoader loader = new PresortRioLoader( buffer );
// @todo review: disable auto-flush - caller will handle flush of the buffer.
// loader.setFlush(false);
-
// add listener to log progress.
loader.addRioLoaderListener( new RioLoaderListener() {
@@ -1013,12 +1017,12 @@
if(source instanceof Reader) {
- loader.loadRdf((Reader) source, baseURL, rdfFormat, parserOptions);
+ loader.loadRdf((Reader) source, baseURL, rdfFormat, defaultGraph, parserOptions);
} else if (source instanceof InputStream) {
loader.loadRdf((InputStream) source, baseURL, rdfFormat,
- parserOptions);
+ defaultGraph, parserOptions);
} else
throw new AssertionError();
@@ -1368,7 +1372,7 @@
dataLoader.loadFiles(totals, 0/* depth */,
fileOrDir.toURI().toURL(), baseURI,
- rdfFormat, filter, true/* endOfBatch */
+ rdfFormat, null, filter, true/* endOfBatch */
);
}
Modified: branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util/Splitter.java
===================================================================
--- branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util/Splitter.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util/Splitter.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -99,7 +99,6 @@
* </pre>
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public class Splitter {
@@ -606,7 +605,6 @@
* Tasks parses an RDF document, writing a new file every N statements.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
protected class ParserTask implements Callable<Void> {
@@ -714,7 +712,7 @@
try {
// run the parser.
new MyLoader(buffer).loadRdf(reader, baseURL,
- defaultRDFFormat, s.parserOptions);
+ defaultRDFFormat, null, s.parserOptions);
} finally {
reader.close();
}
@@ -737,7 +735,6 @@
* {@link StatementBuffer}.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
private
static class MyLoader extends BasicRioLoader implements RDFHandler
@@ -843,7 +840,6 @@
*
* @author <a href="mailto:tho...@us...">Bryan
* Thompson</a>
- * @version $Id$
* @param <S>
*/
private class MyStatementBuffer implements
Modified: branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/AbstractRIOTestCase.java
===================================================================
--- branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/AbstractRIOTestCase.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/AbstractRIOTestCase.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -401,7 +401,7 @@
});
- loader.loadRdf((Reader) reader, baseURI, rdfFormat, options);
+ loader.loadRdf((Reader) reader, baseURI, rdfFormat, null, options);
if (log.isInfoEnabled())
log.info("Done: " + resource);
@@ -681,7 +681,7 @@
loader.loadRdf(new BufferedReader(new InputStreamReader(
new FileInputStream(resource))), baseURI, rdfFormat,
- options);
+ null, options);
if(log.isInfoEnabled())
log.info("End of reparse: nerrors=" + nerrs + ", file="
Modified: branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/EDSAsyncLoader.java
===================================================================
--- branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/EDSAsyncLoader.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/EDSAsyncLoader.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -52,7 +52,6 @@
* </pre>
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public class EDSAsyncLoader {
@@ -161,6 +160,7 @@
valuesInitialCapacity,//
bnodesInitialCapacity,//
RDFFormat.RDFXML, // defaultFormat
+ null, // defaultGraph
parserOptions, // parserOptions
false, // deleteAfter
poolSize, // parserPoolSize,
Modified: branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/TestAsynchronousStatementBufferFactory.java
===================================================================
--- branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/TestAsynchronousStatementBufferFactory.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/TestAsynchronousStatementBufferFactory.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -400,6 +400,7 @@
valuesInitialCapacity,//
bnodesInitialCapacity,//
RDFFormat.RDFXML, // defaultFormat
+ null, // defaultGraph
parserOptions, //
false, // deleteAfter
parallel?5:1, // parserPoolSize,
Modified: branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/stress/LoadClosureAndQueryTest.java
===================================================================
--- branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/stress/LoadClosureAndQueryTest.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/stress/LoadClosureAndQueryTest.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -1204,7 +1204,7 @@
try {
dataLoader.loadFiles(dataDir.toURI().toURL(), null/* baseURI */,
- null/* rdfFormat */, filter);
+ null/* rdfFormat */, null, /* defaultGraph */filter);
} catch (IOException ex) {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <btm...@us...> - 2010-11-11 22:00:49
|
Revision: 3937
http://bigdata.svn.sourceforge.net/bigdata/?rev=3937&view=rev
Author: btmurphy
Date: 2010-11-11 22:00:41 +0000 (Thu, 11 Nov 2010)
Log Message:
-----------
[branch dev-btm]: CHECKPOINT - completed phase 1 of callable executor (client) service smart proxy work. Made changes to allow smart proxy and/or remote service implementations to be started by ServicesManagerService, service re-ordering in RestartPersistentServices because of dependency on txn service and load balancer, added DeferredInitTask to EmbeddedShardService, added the necessary jar files to classpath of shard locator and callable executor services in boot-processes.xml
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/resources/StoreManager.java
branches/dev-btm/bigdata/src/resources/logging/log4j.properties
branches/dev-btm/bigdata-jini/src/java/com/bigdata/boot/config/boot-processes.xml
branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/config/executor.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/config/logging.properties
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/RestartPersistentServices.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServiceConfigurationZNodeMonitorTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/logging.properties
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/shardlocator.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/EmbeddedShardService.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/config/logging.properties
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/config/shard.config
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/master/TestMappedRDFDataLoadMaster.config
branches/dev-btm/src/resources/config/bigdataCluster.config
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/resources/StoreManager.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/resources/StoreManager.java 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/resources/StoreManager.java 2010-11-11 22:00:41 UTC (rev 3937)
@@ -1523,6 +1523,10 @@
// Wait no more than N seconds for discovery
int nWait = 120;
boolean discoveredTxnSrvc = false;
+ if (log.isDebugEnabled()) {
+ log.debug("waiting for transaction "
+ +"service discovery ...");
+ }
for(int i=0; i<nWait; i++) {
if (discoveryMgr.getTransactionService() != null) {
discoveredTxnSrvc = true;
@@ -1531,24 +1535,19 @@
try {
Thread.sleep(1000L);
} catch(InterruptedException ie) { }
+ }
+ if(discoveredTxnSrvc) {
if (log.isDebugEnabled()) {
- log.debug("waiting for transaction "
- +"service discovery");
+ log.debug("StoreManager - discovered "
+ +"transaction service");
}
- if(discoveredTxnSrvc) {
- if (log.isDebugEnabled()) {
- log.debug("discovered transaction service");
- }
- } else {
- log.warn("transaction service unreachable");
- }//endif(discoveredTxnSrvc)
- }//endloop(nWait)
-//BTM if(discoveredTxnSrvc) {
-//BTM com.bigdata.util.Util.printStr("TestBigdataClientRemote.txt","\nStoreManager.start >>> TRANSACTION SERVICE DISCOVERED");
-//BTM }else{
-//BTM com.bigdata.util.Util.printStr("TestBigdataClientRemote.txt","\nStoreManager.start >>> TRANSACTION SERVICE UNREACHABLE\n");
-//BTM }
+log.warn("DISCOVERED TRANSACTION SERVICE");
+ } else {
+ log.warn("StoreManager - transaction "
+ +"service unreachable");
+ }
+
} catch (UnsupportedOperationException ex) {
//BTM com.bigdata.util.Util.printStr("TestBigdataClientRemote.txt","\nStoreManager.start >>> FEDERATION UNAVAILABLE - test case?\n");
log.warn("Federation not available - running in test case?");
Modified: branches/dev-btm/bigdata/src/resources/logging/log4j.properties
===================================================================
--- branches/dev-btm/bigdata/src/resources/logging/log4j.properties 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata/src/resources/logging/log4j.properties 2010-11-11 22:00:41 UTC (rev 3937)
@@ -215,3 +215,4 @@
log4j.logger.com.bigdata.transaction=DEBUG
log4j.logger.com.bigdata.metadata=DEBUG
log4j.logger.com.bigdata.shard=DEBUG
+log4j.logger.com.bigdata.executor=DEBUG
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/boot/config/boot-processes.xml
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/boot/config/boot-processes.xml 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/boot/config/boot-processes.xml 2010-11-11 22:00:41 UTC (rev 3937)
@@ -77,7 +77,7 @@
<javaprop name="java.util.logging.config.file"
value="${bigdata.configDir}/logging/logging.properties"/>
- <property name="java.classpath" value="${bootLauncherClasspath}${:}lib/fastutil.jar${:}lib/dsiutils.jar${:}lib/cweb-extser.jar"/>
+ <property name="java.classpath" value="${bootLauncherClasspath}${:}lib/fastutil.jar${:}lib/dsiutils.jar${:}lib/cweb-extser.jar${:}lib/ctc_utils.jar"/>
<property name="java.app.mainclass"
value="com.bigdata.boot.starter.SingleNonActivatableServiceStarter"/>
<arg value="${bigdata.configDir}/policy/service.policy"/>
@@ -113,7 +113,7 @@
<javaprop name="java.util.logging.config.file"
value="${bigdata.configDir}/logging/logging.properties"/>
- <property name="java.classpath" value="${bootLauncherClasspath}"/>
+ <property name="java.classpath" value="${bootLauncherClasspath}${:}lib/fastutil.jar${:}lib/dsiutils.jar${:}lib/cweb-extser.jar${:}lib/ctc_utils.jar"/>
<property name="java.app.mainclass"
value="com.bigdata.boot.starter.SingleNonActivatableServiceStarter"/>
<arg value="${bigdata.configDir}/policy/service.policy"/>
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/config/executor.config
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/config/executor.config 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/config/executor.config 2010-11-11 22:00:41 UTC (rev 3937)
@@ -11,6 +11,10 @@
import net.jini.core.discovery.LookupLocator;
import net.jini.discovery.LookupDiscoveryManager;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Id;
+
import com.bigdata.util.config.NicUtil;
import com.bigdata.util.config.ConfigDeployUtil;
@@ -106,3 +110,15 @@
(com.bigdata.executor.serverExporterTcpServerEndpoint,
com.bigdata.executor.serverILFactory, false, false);
}
+
+//NOTE: remove once dynamic discovery of zookeeper is added
+org.apache.zookeeper.ZooKeeper {
+
+ zroot = ConfigDeployUtil.getString("federation.name");
+
+ servers = com.bigdata.executor.serverExporterIpAddr+":2888:3888";
+
+ acl = new ACL[] {
+ new ACL(ZooDefs.Perms.ALL, new Id("world", "anyone"))
+ };
+}
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/config/logging.properties
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/config/logging.properties 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/config/logging.properties 2010-11-11 22:00:41 UTC (rev 3937)
@@ -37,5 +37,4 @@
#log4j.logger.com.bigdata.executor=DEBUG
#log4j.logger.com.bigdata.executor.EmbeddedCallableExecutor=DEBUG
-#log4j.logger.com.bigdata.executor.EmbeddedClientIndexStore=DEBUG
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/RestartPersistentServices.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/RestartPersistentServices.java 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/RestartPersistentServices.java 2010-11-11 22:00:41 UTC (rev 3937)
@@ -19,6 +19,9 @@
import com.bigdata.service.jini.JiniFederation;
import com.bigdata.util.InnerCause;
+//BTM - FOR_CLIENT_SERVICE
+import java.util.ArrayList;
+
/**
* Task restarts persistent physical services that should be running on this
* host but which are not discoverable using jini (not found when we query for
@@ -154,7 +157,8 @@
+ BigdataZooDefs.CONFIG;
// these are the ServiceConfigurations.
- final List<String> serviceConfigZNodes;
+//BTM - PRE_CLIENT_SERVICE final List<String> serviceConfigZNodes;
+ List<String> serviceConfigZNodes;
try {
serviceConfigZNodes = zookeeper.getChildren(zconfig, false);
@@ -171,8 +175,109 @@
if (log.isInfoEnabled())
log.info("Considering " + serviceConfigZNodes.size()
+ " service configurations");
-System.out.println("\n*********************************************************");
-System.out.println("*** RestartPersistentServices.runOnce: Considering " + serviceConfigZNodes.size()+ " service configurations");
+System.out.println("\n*** RestartPersistentServices - BEGIN *********************************************************\n");
+for (String serviceConfigZNode : serviceConfigZNodes) {
+ System.out.println("*** RestartPersistentServices.runOnce: serviceConfigZNode = "+ serviceConfigZNode);
+}
+System.out.println("\n*** RestartPersistentServices.runOnce: RE-ORDER - DATA SERVICE LAST\n");
+//BTM - FOR_CLIENT_SERVICE - BEGIN ---------------------------------------
+ // re-order because shard (data) service waits for transaction
+ // service and load balancer service
+ List<String> tmpList = new ArrayList<String>();
+
+ //1. transaction service(s)
+ for (String serviceConfigZNode : serviceConfigZNodes) {
+ if ( (serviceConfigZNode.equals
+ ("com.bigdata.transaction.ServiceImpl")) ||
+ (serviceConfigZNode.equals
+ ("com.bigdata.service.jini.TransactionServer")) )
+ {
+ tmpList.add(serviceConfigZNode);
+ }
+ }
+ //2. load balancer service(s)
+ for (String serviceConfigZNode : serviceConfigZNodes) {
+ if ( serviceConfigZNode.equals
+ ("com.bigdata.loadbalancer.ServiceImpl") ||
+ (serviceConfigZNode.equals
+ ("com.bigdata.service.jini.LoadBalancerServer")) )
+ {
+ tmpList.add(serviceConfigZNode);
+ }
+ }
+ //3. shard locator (metadata) service(s)
+ for (String serviceConfigZNode : serviceConfigZNodes) {
+ if ( serviceConfigZNode.equals
+ ("com.bigdata.metadata.ServiceImpl") ||
+ (serviceConfigZNode.equals
+ ("com.bigdata.service.jini.MetadataServer")) )
+ {
+ tmpList.add(serviceConfigZNode);
+ }
+ }
+ //4. callable executor (client) service(s)
+ for (String serviceConfigZNode : serviceConfigZNodes) {
+ if ( serviceConfigZNode.equals
+ ("com.bigdata.executor.ServiceImpl") ||
+ (serviceConfigZNode.equals
+ ("com.bigdata.service.jini.ClientServer")) )
+ {
+ tmpList.add(serviceConfigZNode);
+ }
+ }
+ //5. shard (data) service(s)
+ for (String serviceConfigZNode : serviceConfigZNodes) {
+ if ( serviceConfigZNode.equals
+ ("com.bigdata.shard.ServiceImpl") ||
+ (serviceConfigZNode.equals
+ ("com.bigdata.service.jini.DataServer")) )
+ {
+ tmpList.add(serviceConfigZNode);
+ }
+ }
+ //6. add anything that's none of the above
+ for (String serviceConfigZNode : serviceConfigZNodes) {
+
+ if ( !(serviceConfigZNode.equals
+ ("com.bigdata.transaction.ServiceImpl")) &&
+ !(serviceConfigZNode.equals
+ ("com.bigdata.service.jini.TransactionServer")) &&
+
+ !(serviceConfigZNode.equals
+ ("com.bigdata.loadbalancer.ServiceImpl")) &&
+ !(serviceConfigZNode.equals
+ ("com.bigdata.service.jini.LoadBalancerServer")) &&
+
+ !(serviceConfigZNode.equals
+ ("com.bigdata.metadata.ServiceImpl")) &&
+ !(serviceConfigZNode.equals
+ ("com.bigdata.service.jini.MetadataServer")) &&
+
+ !(serviceConfigZNode.equals
+ ("com.bigdata.executor.ServiceImpl")) &&
+ !(serviceConfigZNode.equals
+ ("com.bigdata.service.jini.ClientServer")) &&
+
+ !(serviceConfigZNode.equals
+ ("com.bigdata.shard.ServiceImpl")) &&
+ !(serviceConfigZNode.equals
+ ("com.bigdata.service.jini.DataServer")) )
+ {
+ tmpList.add(serviceConfigZNode);
+ }
+ }
+ if (tmpList.size() == serviceConfigZNodes.size()) {
+ serviceConfigZNodes = tmpList;
+ } else {
+ log.warn("reordered list size ["+tmpList.size()
+ +"] != serviceConfigZNodes size ["
+ +serviceConfigZNodes.size()+"]");
+ }
+System.out.println("*** RestartPersistentServices.runOnce: Considering " + serviceConfigZNodes.size()+ " service configurations\n");
+for (String serviceConfigZNode : serviceConfigZNodes) {
+ System.out.println("*** RestartPersistentServices.runOnce AFTER RE-ORDER: serviceConfigZNode = "+ serviceConfigZNode);
+}
+//BTM - FOR_CLIENT_SERVICE - END -------------------------------------
for (String serviceConfigZNode : serviceConfigZNodes) {
@@ -253,7 +358,7 @@
}
}
-System.out.println("*********************************************************\n");
+System.out.println("\n*** RestartPersistentServices - END *********************************************************\n");
// Success.
return true;
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServiceConfigurationZNodeMonitorTask.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServiceConfigurationZNodeMonitorTask.java 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServiceConfigurationZNodeMonitorTask.java 2010-11-11 22:00:41 UTC (rev 3937)
@@ -323,7 +323,7 @@
if (config.serviceCount != children.size()) {
// adjust the #of logical service instances (blocks).
-System.out.println("GGGG ServiceConfigurationZNodeMonitorTask.runWithLock: *** NEW LOGICAL SERVICE TASK");
+System.out.println("GGGG ServiceConfigurationZNodeMonitorTask.runWithLock: *** NEW LOGICAL SERVICE TASK >>> "+serviceConfigZPath);
config.newLogicalServiceTask(fed, listener, serviceConfigZPath,
children).call();
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServiceConfiguration.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServiceConfiguration.java 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServiceConfiguration.java 2010-11-11 22:00:41 UTC (rev 3937)
@@ -494,7 +494,6 @@
* if the service detectably did not start.
*/
public V call() throws Exception {
-System.out.println("*** ServiceConfiguration: call() [className="+className+"]");
if (log.isInfoEnabled())
log.info("config: " + this);
@@ -532,7 +531,9 @@
TimeUnit.MILLISECONDS);
// attempt to detect a service start failure.
+System.out.println("\n*** ServiceConfiguration: call() [className="+className+"] >>> awaitServiceStart [timeout="+timeout+" ms] - BEGIN\n");
awaitServiceStart(processHelper, timeout, TimeUnit.MILLISECONDS);
+System.out.println("\n*** ServiceConfiguration: call() [className="+className+"] >>> awaitServiceStart [timeout="+timeout+" ms] - END\n");
} catch (InterruptedException ex) {
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/logging.properties
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/logging.properties 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/logging.properties 2010-11-11 22:00:41 UTC (rev 3937)
@@ -37,5 +37,4 @@
#log4j.logger.com.bigdata.metadata=DEBUG
#log4j.logger.com.bigdata.metadata.EmbeddedShardLocator=DEBUG
-#log4j.logger.com.bigdata.journal.EmbeddedIndexStore=DEBUG
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/shardlocator.config
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/shardlocator.config 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/shardlocator.config 2010-11-11 22:00:41 UTC (rev 3937)
@@ -11,6 +11,10 @@
import net.jini.core.discovery.LookupLocator;
import net.jini.discovery.LookupDiscoveryManager;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Id;
+
import com.bigdata.util.config.NicUtil;
import com.bigdata.util.config.ConfigDeployUtil;
@@ -89,3 +93,15 @@
(com.bigdata.metadata.serverExporterTcpServerEndpoint,
com.bigdata.metadata.serverILFactory, false, false);
}
+
+//NOTE: remove once dynamic discovery of zookeeper is added
+org.apache.zookeeper.ZooKeeper {
+
+ zroot = ConfigDeployUtil.getString("federation.name");
+
+ servers = com.bigdata.metadata.serverExporterIpAddr+":2888:3888";
+
+ acl = new ACL[] {
+ new ACL(ZooDefs.Perms.ALL, new Id("world", "anyone"))
+ };
+}
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/EmbeddedShardService.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/EmbeddedShardService.java 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/EmbeddedShardService.java 2010-11-11 22:00:41 UTC (rev 3937)
@@ -156,12 +156,14 @@
//BTM - BEGIN - fields from AbstractFederation -------------------------------
private ScheduledFuture<?> eventTaskFuture;
private ScheduledFuture<?> lbsReportingTaskFuture;
+ private long lbsReportingPeriod;
private AbstractHTTPD httpServer;
private String httpServerUrl;//URL used to access the httpServer
private int httpdPort;
//BTM - END - fields from AbstractFederation -------------------------------
-String dbgFlnm="EmbeddedShardService.out";
+ private ScheduledFuture deferredInitTaskFuture;
+ private boolean deferredInitDone = false;
protected EmbeddedShardService
(final UUID serviceUUID,
@@ -236,116 +238,45 @@
maxStaleLocatorRetries,
logger,
this.properties);
- String httpServerPath =
- (this.localResources).getServiceCounterPathPrefix();
- try {
- this.httpServerUrl =
- "http://"
- +AbstractStatisticsCollector.fullyQualifiedHostName
- +":"+this.httpdPort+"/?path="
- +URLEncoder.encode(httpServerPath, "UTF-8");
- } catch(java.io.UnsupportedEncodingException e) {
- logger.warn("failed to initialize httpServerUrl", e);
- }
-System.out.println("\nEmbeddedShardService >>> NEW StoreManager - BEGIN");
- this.resourceMgr =
- new ShardResourceManager(this,
- this.discoveryMgr,
- this.localResources,
- this.indexMgr,
- this.properties);
-System.out.println("\nEmbeddedShardService >>> NEW StoreManager - END");
+ this.lbsReportingPeriod = lbsReportingPeriod;
- this.localTransactionMgr = new LocalTransactionManager(discoveryMgr);
- this.concurrencyMgr =
- new ConcurrencyManager(this.properties,
- this.localTransactionMgr,
- this.resourceMgr);
- //WARN: circular refs
- (this.resourceMgr).setConcurrencyManager(this.concurrencyMgr);
- (this.indexMgr).setConcurrencyManager(this.concurrencyMgr);
+ // Note that this service employs a StoreManager (in the
+ // ResourceManager) whose creation depends on the existence
+ // of a transaction service. Additionally, this service
+ // also employs counters dependent on the existence of
+ // a load balancer to which the counters send events.
+ // Unfortunately, when the ServicesManagerService is used
+ // to start this service, these dependencies can cause
+ // problems for the ServicesManagerService. This is because
+ // the order the services are started by the ServicesManagerService
+ // can be random, and if this service is the first service
+ // the ServicesManagerService attempts to start (or whose
+ // starting is attempted before the transaction service
+ // and/or the load balancer), then unless this service
+ // returns an indication to the ServicesManagerService that
+ // it has successfully started within the time period
+ // expected, the ServicesManagerService will declare that
+ // this service is faulty and kill the process in which
+ // this service was started. To address this issue, this
+ // service executes an instance of DeferredInitTask to
+ // create the ResourceManager and set up the counters and
+ // events asynchronously; which allows the transaction
+ // service and load balancer to be started and discovered
+ // after this service has been started by the
+ // ServicesManagerService.
-//BTM - from AbstractFederation constructor and addScheduledTask
-
- //start event queue/sender task (sends events every 2 secs)
-
- long sendEventsDelay = 100L;//one-time initial delay
- long sendEventsPeriod = 2000L;
- this.eventTaskFuture =
- (localResources.getScheduledExecutor()).scheduleWithFixedDelay
- (localResources.getEventQueueSender(),
- sendEventsDelay,
- sendEventsPeriod,
- TimeUnit.MILLISECONDS);
-
-//BTM - from AbstractFederation - start deferred tasks
-
- //start task to report counters to the load balancer
-
- LoadBalancerReportingTask lbsReportingTask =
- new LoadBalancerReportingTask(this.resourceMgr,
- this.concurrencyMgr,
- this.localResources,
- this.discoveryMgr,
- logger);
- this.lbsReportingTaskFuture =
- ((this.localResources).getScheduledExecutor())
+ deferredInitDone = deferredInit();
+ if (!deferredInitDone) {
+ DeferredInitTask deferredInitTask = new DeferredInitTask();
+ this.deferredInitTaskFuture =
+ ((this.localResources).getScheduledExecutor())
.scheduleWithFixedDelay
- (lbsReportingTask,
- lbsReportingPeriod,//initial delay
- lbsReportingPeriod,
+ (deferredInitTask,
+ 20L*1000L,//initial delay
+ 30L*1000L,//period
TimeUnit.MILLISECONDS);
-
- //start an http daemon from which interested parties can query
- //counter and/or statistics information with http get commands
-
- try {
- httpServer =
- new HttpReportingServer(this.httpdPort,
- this.resourceMgr,
- this.concurrencyMgr,
- this.localResources,
- logger);
- } catch (IOException e) {
- logger.error("failed to start http server "
- +"[port="+this.httpdPort
- +", path="+httpServerPath+"]", e);
- return;
}
- if(httpServer != null) {
- if( logger.isDebugEnabled() ) {
- logger.debug("started http daemon "
- +"[access URL="+this.httpServerUrl+"]");
- }
- // add counter reporting the access url to load balancer
- ((this.localResources).getServiceCounterSet())
- .addCounter
- (IServiceCounters.LOCAL_HTTPD,
- new OneShotInstrument<String>(this.httpServerUrl));
- }
-
-//BTM - BEGIN ScaleOutIndexManager
-//BTM - The call to embeddedIndexStore.didStart was previously
-//BTM - commented out during the data service conversion. But
-//BTM - the method didStart() on the original EmbeddedIndexStore and
-//BTM - AbstractFederation calls the private method setupCounters;
-//BTM - which seems to be important for at least the shard (data)
-//BTM - service. The tests still passed without calling that method,
-//BTM - but we should consider calling it at this point (the problem
-//BTM - is that it waits on the resource manager to finish
-//BTM - initializing, which waits on the transaction service to be
-//BTM - discovered). Setting up these counters seem to be important
-//BTM - only for the shard (data) service rather than the other
-//BTM - services. So we should consider adding setupCounters to this
-//BTM - class, and calling it here instead of calling
-//BTM - embeddedIndexStore.didStart() or AbstractFederation.didStart().
-//BTM -
-//BTM embeddedIndexStore.didStart();
-
- setupCounters();
-
-//BTM - END ScaleOutIndexManager
}
// Required by Service interface
@@ -394,6 +325,11 @@
logger.warn("SSSS SHARD SERVICE EmbeddedShardService.shutdown");
if (!isOpen()) return;
+ //false ==> allow in-progress tasks to complete
+ if (deferredInitTaskFuture != null) {
+ deferredInitTaskFuture.cancel(false);
+ }
+
if (concurrencyMgr != null) {
concurrencyMgr.shutdown();
}
@@ -404,9 +340,12 @@
resourceMgr.shutdown();
}
- //false ==> allow in-progress tasks to complete
- lbsReportingTaskFuture.cancel(false);
- eventTaskFuture.cancel(false);
+ if (lbsReportingTaskFuture != null) {
+ lbsReportingTaskFuture.cancel(false);
+ }
+ if (eventTaskFuture != null) {
+ eventTaskFuture.cancel(false);
+ }
if (indexMgr != null) indexMgr.destroy();
if (localResources != null) {
@@ -1026,6 +965,119 @@
return new File(resourceMgr.getDataDir(), "httpd.url");
}
+ private boolean deferredInit() {
+
+ // StoreManager depends on the transaction service
+ if (discoveryMgr.getTransactionService() == null) return false;
+
+ if (this.resourceMgr == null) {
+System.out.println("\nEmbeddedShardService >>> NEW StoreManager - BEGIN");
+ this.resourceMgr =
+ new ShardResourceManager(this,
+ this.discoveryMgr,
+ this.localResources,
+ this.indexMgr,
+ this.properties);
+System.out.println("\nEmbeddedShardService >>> NEW StoreManager - END");
+
+ this.localTransactionMgr =
+ new LocalTransactionManager(discoveryMgr);
+ this.concurrencyMgr =
+ new ConcurrencyManager(this.properties,
+ this.localTransactionMgr,
+ this.resourceMgr);
+ //WARN: circular refs
+ (this.resourceMgr).setConcurrencyManager(this.concurrencyMgr);
+ (this.indexMgr).setConcurrencyManager(this.concurrencyMgr);
+ }
+
+ // Events and counters depend on the load balancer
+ if (discoveryMgr.getLoadBalancerService() == null) return false;
+
+//BTM - from AbstractFederation - start deferred tasks
+
+ //start task to report counters to the load balancer
+
+ LoadBalancerReportingTask lbsReportingTask =
+ new LoadBalancerReportingTask(this.resourceMgr,
+ this.concurrencyMgr,
+ this.localResources,
+ this.discoveryMgr,
+ logger);
+ this.lbsReportingTaskFuture =
+ ((this.localResources).getScheduledExecutor())
+ .scheduleWithFixedDelay
+ (lbsReportingTask,
+ lbsReportingPeriod,//initial delay
+ lbsReportingPeriod,
+ TimeUnit.MILLISECONDS);
+
+ //start an http daemon from which interested parties can query
+ //counter and/or statistics information with http get commands
+
+ String httpServerPath =
+ (this.localResources).getServiceCounterPathPrefix();
+ try {
+ this.httpServerUrl =
+ "http://"
+ +AbstractStatisticsCollector.fullyQualifiedHostName
+ +":"+this.httpdPort+"/?path="
+ +URLEncoder.encode(httpServerPath, "UTF-8");
+ } catch(java.io.UnsupportedEncodingException e) {
+ logger.warn("failed to initialize httpServerUrl", e);
+ }
+
+
+ try {
+ httpServer =
+ new HttpReportingServer(this.httpdPort,
+ this.resourceMgr,
+ this.concurrencyMgr,
+ this.localResources,
+ logger);
+ } catch (IOException e) {
+ logger.error("failed to start http server "
+ +"[port="+this.httpdPort
+ +", path="+httpServerPath+"]", e);
+ return false;
+ }
+ if(httpServer != null) {
+ if( logger.isDebugEnabled() ) {
+ logger.debug("started http daemon "
+ +"[access URL="+this.httpServerUrl+"]");
+ }
+ // add counter reporting the access url to load balancer
+ ((this.localResources).getServiceCounterSet())
+ .addCounter
+ (IServiceCounters.LOCAL_HTTPD,
+ new OneShotInstrument<String>(this.httpServerUrl));
+ }
+
+//BTM - BEGIN ScaleOutIndexManager Note
+//BTM - The call to embeddedIndexStore.didStart was previously
+//BTM - commented out during the data service conversion. But
+//BTM - the method didStart() on the original EmbeddedIndexStore and
+//BTM - AbstractFederation calls the private method setupCounters;
+//BTM - which seems to be important for at least the shard (data)
+//BTM - service. The tests still passed without calling that method,
+//BTM - but we should consider calling it at this point (the problem
+//BTM - is that it waits on the resource manager to finish
+//BTM - initializing, which waits on the transaction service to be
+//BTM - discovered). Setting up these counters seem to be important
+//BTM - only for the shard (data) service rather than the other
+//BTM - services. So we should consider adding setupCounters to this
+//BTM - class, and calling it here instead of calling
+//BTM - embeddedIndexStore.didStart() or AbstractFederation.didStart().
+//BTM -
+//BTM embeddedIndexStore.didStart();
+
+ setupCounters();
+
+//BTM - END ScaleOutIndexManager Note
+
+ return true;
+ }
+
private void setupLoggingContext() {
try {
@@ -1120,8 +1172,37 @@
}
}
-//BTM - see the note at the end of this class' constructor
+ class DeferredInitTask implements Runnable {
+ public DeferredInitTask() { }
+
+ public void run() {
+ try {
+ if (!deferredInitDone) {
+System.out.println("\n*** EmbededShardService#DeferredInitTask: DO DEFERRED INIT \n");
+ deferredInitDone = deferredInit();
+ } else {
+System.out.println("\n*** EmbededShardService#DeferredInitTask: DEFERRED INIT DONE >>> CANCELLING TASK\n");
+ deferredInitDone = true;
+ if (deferredInitTaskFuture != null) {
+ deferredInitTaskFuture.cancel(false);
+ deferredInitTaskFuture = null;
+ }
+ }
+ } catch (Throwable t) {
+System.out.println("\n*** EmbededShardService#DeferredInitTask: EXCEPTION >>> "+t+"\n");
+ logger.error("deferred initialization failure", t);
+ deferredInitDone = true;
+ if (deferredInitTaskFuture != null) {
+ deferredInitTaskFuture.cancel(false);
+ deferredInitTaskFuture = null;
+ }
+ }
+ }
+ }
+
+//BTM - see the note at the end of the deferredInit method
+
/**
* Sets up shard service specific counters.
* <p>
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/config/logging.properties
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/config/logging.properties 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/config/logging.properties 2010-11-11 22:00:41 UTC (rev 3937)
@@ -37,5 +37,4 @@
#log4j.logger.com.bigdata.shard=DEBUG
#log4j.logger.com.bigdata.shard.EmbeddedShardService=DEBUG
-#log4j.logger.com.bigdata.journal.EmbeddedIndexStore=DEBUG
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/config/shard.config
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/config/shard.config 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/config/shard.config 2010-11-11 22:00:41 UTC (rev 3937)
@@ -11,6 +11,10 @@
import net.jini.core.discovery.LookupLocator;
import net.jini.discovery.LookupDiscoveryManager;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Id;
+
import com.bigdata.util.config.NicUtil;
import com.bigdata.util.config.ConfigDeployUtil;
@@ -89,3 +93,15 @@
(com.bigdata.shard.serverExporterTcpServerEndpoint,
com.bigdata.shard.serverILFactory, false, false);
}
+
+//NOTE: remove once dynamic discovery of zookeeper is added
+org.apache.zookeeper.ZooKeeper {
+
+ zroot = ConfigDeployUtil.getString("federation.name");
+
+ servers = com.bigdata.shard.serverExporterIpAddr+":2888:3888";
+
+ acl = new ACL[] {
+ new ACL(ZooDefs.Perms.ALL, new Id("world", "anyone"))
+ };
+}
Modified: branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/master/TestMappedRDFDataLoadMaster.config
===================================================================
--- branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/master/TestMappedRDFDataLoadMaster.config 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/master/TestMappedRDFDataLoadMaster.config 2010-11-11 22:00:41 UTC (rev 3937)
@@ -137,7 +137,7 @@
/* Template for matching the services to which the clients will be
* distributed for execution. Normally you will specify
- * IClientService as the interface to be discovered. While it is
+ * CallableExecutor as the interface to be discovered. While it is
* possible to run tasks on an IDataService or even an
* IMetadataService since they both implement IRemoteExecutor, it
* is generally discouraged unless the tasks require explicit
@@ -148,7 +148,8 @@
new ServiceTemplate(
null, //serviceID
new Class[]{
- com.bigdata.service.IClientService.class
+//BTM com.bigdata.service.IClientService.class
+ com.bigdata.service.CallableExecutor.class
},
null // attributes
),
Modified: branches/dev-btm/src/resources/config/bigdataCluster.config
===================================================================
--- branches/dev-btm/src/resources/config/bigdataCluster.config 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/src/resources/config/bigdataCluster.config 2010-11-11 22:00:41 UTC (rev 3937)
@@ -630,14 +630,15 @@
"org.apache.zookeeper.server.quorum.QuorumPeerMain",
//BTM "com.bigdata.service.jini.TransactionServer",
//BTM "com.bigdata.service.jini.MetadataServer",
- "com.bigdata.service.jini.DataServer",
+//BTM "com.bigdata.service.jini.DataServer",
//BTM "com.bigdata.service.jini.LoadBalancerServer",
- "com.bigdata.service.jini.ClientServer",
+//BTM "com.bigdata.service.jini.ClientServer",
"com.bigdata.transaction.ServiceImpl",
"com.bigdata.metadata.ServiceImpl",
-//BTM "com.bigdata.shard.ServiceImpl",
-"com.bigdata.loadbalancer.ServiceImpl"
+"com.bigdata.shard.ServiceImpl",
+"com.bigdata.loadbalancer.ServiceImpl",
+"com.bigdata.executor.ServiceImpl"
};
@@ -1340,6 +1341,36 @@
// log4j = "file:@NAS@/dist/bigdata/var/config/logging/loadbalancer-logging.properties";
}
+
+com.bigdata.executor.ServiceImpl {
+
+ constraints = new IServiceConstraint[] {
+ new JiniRunningConstraint(),
+ new ZookeeperRunningConstraint(),
+ new HostAllowConstraint(bigdata.cs),
+ new MaxClientServicesPerHostConstraint(bigdata.maxClientServicePerHost)
+ };
+
+ args = new String[]{
+ "-Xmx1600m", // was 800m
+ //"-XX:-UseGCOverheadLimit",//keeps VM alive even when memory starved
+ "-XX:+UseParallelOldGC",
+ //"-XX:ParallelGCThreads=8",
+
+ "-Djava.util.logging.config.file=@NAS@/dist/bigdata/var/config/logging/executor-logging.properties",
+ "-Dlog4j.configuration=@NAS@/dist/bigdata/var/config/logging/executor-logging.properties",
+ "-Dlog4j.primary.configuration=@NAS@/dist/bigdata/var/config/logging/executor-logging.properties",
+ "-DusingServiceConfiguration=true",
+ "-Dbigdata.logDir=@NAS@/dist/bigdata/var/log",
+ "-DappHome=@APP_HOME@",
+ "-Dconfig=@NAS@/dist/bigdata/var/config/jini/executor.config"
+ };
+
+ serviceCount = bigdata.clientServiceCount;
+
+ properties = new NV[] {
+ };
+}
//BTM - END
/**
@@ -1959,7 +1990,7 @@
/* Template for matching the services to which the clients will be
* distributed for execution. Normally you will specify
- * IClientService as the interface to be discovered. While it is
+ * CallableExecutor as the interface to be discovered. While it is
* possible to run tasks on a shard service or even a shard
* locator service since they both implement ShardManagement, it
* is generally discouraged unless the tasks require explicit
@@ -1970,7 +2001,8 @@
new ServiceTemplate(
null, //serviceID
new Class[]{
- com.bigdata.service.IClientService.class
+//BTM com.bigdata.service.IClientService.class
+ com.bigdata.service.CallableExecutor.class
},
null // attributes
),
@@ -2065,7 +2097,7 @@
/* Template for matching the services to which the clients will be
* distributed for execution. Normally you will specify
- * IClientService as the interface to be discovered. While it is
+ * CallableExecutor as the interface to be discovered. While it is
* possible to run tasks on a shard service or even a shard
* locator service since they both implement ShardManagement, it
* is generally discouraged unless the tasks require explicit
@@ -2076,7 +2108,8 @@
new ServiceTemplate(
null, //serviceID
new Class[]{
- com.bigdata.service.IClientService.class
+//BTM com.bigdata.service.IClientService.class
+ com.bigdata.service.CallableExecutor.class
},
null // attributes
),
@@ -2261,7 +2294,7 @@
/* Template for matching the services to which the clients will be
* distributed for execution. Normally you will specify
- * IClientService as the interface to be discovered. While it is
+ * CallableExecutor as the interface to be discovered. While it is
* possible to run tasks on a shard service or even a shard
* locator service since they both implement ShardManagement, it
* is generally discouraged unless the tasks require explicit
@@ -2272,7 +2305,8 @@
new ServiceTemplate(
null, //serviceID
new Class[]{
- com.bigdata.service.IClientService.class
+//BTM com.bigdata.service.IClientService.class
+ com.bigdata.service.CallableExecutor.class
},
null), // attributes
null // filter
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <btm...@us...> - 2010-11-22 17:42:24
|
Revision: 3971
http://bigdata.svn.sourceforge.net/bigdata/?rev=3971&view=rev
Author: btmurphy
Date: 2010-11-22 17:42:16 +0000 (Mon, 22 Nov 2010)
Log Message:
-----------
[branch dev-btm]: CHECKPOINT - fixes and additions to QuorumPeerService ServiceImpl class and related config to support starting and stopping multiple instances, added new QuorumPeerServiceTest to verify the fixes and additions
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractCounterSet.java
branches/dev-btm/bigdata/src/resources/logging/log4j.properties
branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/Constants.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/config/quorum.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/ConfigDeployUtil.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/default-deploy.properties
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/deploy.properties
branches/dev-btm/build.xml
Added Paths:
-----------
branches/dev-btm/bigdata-jini/lib/apache/zookeeper-3.3.2.jar
branches/dev-btm/bigdata-jini/src/test/com/bigdata/quorum/
branches/dev-btm/bigdata-jini/src/test/com/bigdata/quorum/QuorumPeerServiceTest.java
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractCounterSet.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractCounterSet.java 2010-11-19 18:12:50 UTC (rev 3970)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractCounterSet.java 2010-11-22 17:42:16 UTC (rev 3971)
@@ -217,6 +217,8 @@
strBuf.append( new String(new byte[] {pathBytes[i]}) );
}
}
+
+//BTM - workaround for http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6935535
logger.warn("***** AbstractCounterSet.getPath: CONTAINS SLASH-SLASH: path CONVERTED = "+strBuf.toString());
}
int slashSlashIndex = path.indexOf("//");
Modified: branches/dev-btm/bigdata/src/resources/logging/log4j.properties
===================================================================
--- branches/dev-btm/bigdata/src/resources/logging/log4j.properties 2010-11-19 18:12:50 UTC (rev 3970)
+++ branches/dev-btm/bigdata/src/resources/logging/log4j.properties 2010-11-22 17:42:16 UTC (rev 3971)
@@ -211,8 +211,13 @@
#log4j.logger.com.bigdata.boot.launcher.ConfigReaderUnitTest=DEBUG
#log4j.logger.com.bigdata.process.ProcessConfigXmlHandlerTest=DEBUG
-log4j.logger.com.bigdata.loadbalancer=DEBUG
-log4j.logger.com.bigdata.transaction=DEBUG
-log4j.logger.com.bigdata.metadata=DEBUG
-log4j.logger.com.bigdata.shard=DEBUG
-log4j.logger.com.bigdata.executor=DEBUG
+# For the quorum server tests
+log4j.logger.com.bigdata.quorum.QuorumPeerServiceTest=INFO
+
+# For smart proxy implementations of the services
+#log4j.logger.com.bigdata.loadbalancer=DEBUG
+#log4j.logger.com.bigdata.transaction=DEBUG
+#log4j.logger.com.bigdata.metadata=DEBUG
+#log4j.logger.com.bigdata.shard=DEBUG
+#log4j.logger.com.bigdata.executor=DEBUG
+#log4j.logger.com.bigdata.quorum=DEBUG
Added: branches/dev-btm/bigdata-jini/lib/apache/zookeeper-3.3.2.jar
===================================================================
(Binary files differ)
Property changes on: branches/dev-btm/bigdata-jini/lib/apache/zookeeper-3.3.2.jar
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/Constants.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/Constants.java 2010-11-19 18:12:50 UTC (rev 3970)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/Constants.java 2010-11-22 17:42:16 UTC (rev 3971)
@@ -38,4 +38,9 @@
/* Name of this component; used in config entry retrieval and the logger.*/
String COMPONENT_NAME = ((Constants.class).getPackage()).getName();
String F_SEP = System.getProperty("file.separator");
+
+ // Time (in seconds) to wait for discovery of other peers on 1st start up
+ long LOWER_BOUND_PEER_DISCOVERY_PERIOD = 1L;
+ long UPPER_BOUND_PEER_DISCOVERY_PERIOD = Long.MAX_VALUE;
+ long DEFAULT_PEER_DISCOVERY_PERIOD = 5L*60L;
}
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/ServiceImpl.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/ServiceImpl.java 2010-11-19 18:12:50 UTC (rev 3970)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/ServiceImpl.java 2010-11-22 17:42:16 UTC (rev 3971)
@@ -31,6 +31,7 @@
import com.bigdata.attr.ServiceInfo;
import com.bigdata.service.QuorumPeerService;
import com.bigdata.service.QuorumPeerService.QuorumPeerData;
+import com.bigdata.util.EntryUtil;
import com.bigdata.util.Util;
import com.bigdata.util.config.ConfigDeployUtil;
import com.bigdata.util.config.LogUtil;
@@ -332,12 +333,23 @@
joinMgr = new JoinManager(outerProxy, serviceAttrs, serviceId, ldm,
null, config);
- if((peerState.getNQuorumPeers() > 1L) && (peerState.getPeerId() == 0L))
- {
- //discover all other peers to determine peer ids and server info
- //so the entire quorum peer ensemble can be initialized & persisted
- initQuorumPeerData(peerState, configStateInfo);
+ // If not standalone and if this is the very first time this
+ // service instance has started (that is, it has not previously
+ // persisted any state related to the other peers in the
+ // ensemble), then attempt to discover all other peers in the
+ // ensemble to determine peer ids and server info so the entire
+ // quorum peer ensemble can be initialized & persisted.
+ if((peerState.getNQuorumPeers() > 1L)&&(peerState.getPeerId() == 0L)) {
+ long waitPeriod =
+ Config.getLongEntry(config,
+ COMPONENT_NAME,
+ "peerDiscoveryPeriod",
+ DEFAULT_PEER_DISCOVERY_PERIOD,
+ LOWER_BOUND_PEER_DISCOVERY_PERIOD,
+ UPPER_BOUND_PEER_DISCOVERY_PERIOD);
+ initQuorumPeerData(peerState, configStateInfo, waitPeriod);
+
//update the peerId of the QuorumPeerAttr
QuorumPeerAttr tmplVal = new QuorumPeerAttr();
QuorumPeerAttr changeVal = new QuorumPeerAttr();
@@ -354,6 +366,18 @@
myidOut.flush();
myidOut.close();
+ // zookeeperJmxLog4j item is not part of the persisted state
+ Boolean defaultJmxLog4j =
+ ConfigDeployUtil.getBoolean("federation.zookeeperJmxLog4j");
+ boolean zookeeperJmxLog4j = (Boolean)config.getEntry
+ (COMPONENT_NAME,
+ "zookeeperJmxLog4j",
+ Boolean.class,
+ defaultJmxLog4j);
+ if (!zookeeperJmxLog4j) {
+ System.setProperty("zookeeper.jmx.log4j.disable", "true");
+ }
+
this.quorumPeerMainTaskExecutor = Executors.newFixedThreadPool(1);
this.quorumPeerMainTask = new QuorumPeerMainTask(configStateInfo);
this.quorumPeerMainTaskExecutor.execute(this.quorumPeerMainTask);
@@ -374,7 +398,8 @@
}
private boolean initQuorumPeerData(QuorumPeerState peerState,
- ConfigStateInfo configStateInfo)
+ ConfigStateInfo configStateInfo,
+ long peerDiscoveryPeriod)
throws IOException, ConfigurationException
{
int nPeersInEnsemble = peerState.getNQuorumPeers();
@@ -387,21 +412,26 @@
Class[] peerTmplTypes = new Class[] { QuorumPeerService.class };
QuorumPeerAttr peerAttr = new QuorumPeerAttr();
- //match on all ports
- peerAttr.peerPort = peerState.getPeerPort();
- peerAttr.electionPort = peerState.getElectionPort();
- peerAttr.clientPort = peerState.getClientPort();
+
+//BTM Ports can all be different (which helps when running multiple
+//BTM servers on the same node), so no need to do exact matching
+//BTM on the client, peer, and election ports. Thus, discover all
+//BTM servers in the ensemble by federation group and service type,
+//BTM and wildcard the port values.
+
+//BTM peerAttr.peerPort = peerState.getPeerPort();
+//BTM peerAttr.electionPort = peerState.getElectionPort();
+//BTM peerAttr.clientPort = peerState.getClientPort();
+
Entry[] peerTmplAttrs = new Entry[] { peerAttr };
-
ServiceTemplate peerTmpl = new ServiceTemplate(peerTmplId,
peerTmplTypes,
peerTmplAttrs);
-
- long nWait = 5L*60L*1000L;//wait 5 minutes, then give up
ServiceItem[] peerItems = null;
try {
peerItems = sdm.lookup(peerTmpl, nPeersInEnsemble,
- nPeersInEnsemble, null, nWait);
+ nPeersInEnsemble, null,
+ peerDiscoveryPeriod);
if((peerItems == null) || (peerItems.length < nPeersInEnsemble)) {
return false;
}
@@ -411,34 +441,63 @@
// Found all peers, including self. Set peerId based on serviceId:
// "smallest" serviceId is set to 1, next "smallest" set to 2, etc.
+ // unless a discovered service already has a non-zero peerId field
+ // in its QuroumPeerAttr attribute.
//
- // Use TreeSet to order the proxyId's from lowest to highest
+ // Use TreeMap to order the proxyId's from lowest to highest
// (the UUID elements provide a compareTo method for consistent
// ordering).
- Set<UUID> orderedProxyIdSet = new TreeSet<UUID>();
- for(int i=0; i<peerItems.length; i++) {
- orderedProxyIdSet.add
- (((QuorumPeerService)(peerItems[i].service)).getServiceUUID());
- }
+ //
+ // Also, while populating the ordered map, determine this service's
+ // own proxyId and peerId so they can be used later when constructing
+ // and persisting the map of QuorumPeerData.
+
UUID thisProxyId = peerState.getProxyId();
if(thisProxyId == null) {
throw new NullPointerException("initQuorumPeerData: "
+"null proxyId from peerState");
}
+ Long thisPeerId = 0L;//will replace this with non-zero below
- // Determine this service's own peerId and create an ordered map
- // that maps each service's proxyId to its corresponding peerId
- // so that the QuorumPeerData map can be constructed and persisted.
- long thisPeerId = 0L;
+ // Populate the map with the ordered proxy id keys and either
+ // a non-zero peer id (indicating the discovered service had a
+ // previously initialized - and persisted - peerId) or 0 to
+ // indicate that the discovered service has been started for
+ // the very first time
+
Map<UUID, Long> orderedPeerIdMap = new TreeMap<UUID, Long>();
- Iterator<UUID> proxyItr = orderedProxyIdSet.iterator();
- for(long peerId=1; proxyItr.hasNext(); peerId++) {
- UUID nextProxyId = proxyItr.next();
- orderedPeerIdMap.put(nextProxyId, peerId);
- if( thisProxyId.equals(nextProxyId) ) {
- thisPeerId = peerId;
+ for(int i=0; i<peerItems.length; i++) {
+ UUID proxyId =
+ ((QuorumPeerService)(peerItems[i].service)).getServiceUUID();
+ Entry[] attrs = peerItems[i].attributeSets;
+ QuorumPeerAttr quorumPeerAttr =
+ EntryUtil.getEntryByType(attrs, QuorumPeerAttr.class);
+ Long peerId = (quorumPeerAttr == null ? 0L:quorumPeerAttr.peerId);
+ orderedPeerIdMap.put(proxyId, peerId);
+ logger.debug("PUT peerId >>> ["+proxyId+", "+peerId+"]");
+ }
+
+ // Replace any 0-valued peer ids from above with a non-zero value
+ // in the correct order.
+
+ Set<Map.Entry<UUID, Long>> orderedSet = orderedPeerIdMap.entrySet();
+ Iterator<Map.Entry<UUID, Long>> itr = orderedSet.iterator();
+ for(Long peerIdCntr=1L; itr.hasNext(); peerIdCntr++ ) {
+ Map.Entry<UUID, Long> pair = itr.next();
+ UUID curProxyId = pair.getKey();
+ Long curPeerId = pair.getValue();
+ if (curPeerId == 0L) {
+ curPeerId = peerIdCntr;
+ orderedPeerIdMap.put(curProxyId, curPeerId);//replace
+ logger.debug("REPLACE peerId >>> "
+ +"["+curProxyId+", "+curPeerId+"]");
}
+ if( thisProxyId.equals(curProxyId) ) {//determine own peerId
+ thisPeerId = curPeerId;
+ }
}
+
+ //verify this service's peerId was indeed determined above
if(thisPeerId == 0) return false;
peerState.setPeerId(thisPeerId);
@@ -462,6 +521,8 @@
( new InetSocketAddress(peerAddress, peerPort) );
peerData.setElectionAddress
( new InetSocketAddress(peerAddress, electionPort) );
+
+ peerDataMap.put(peerId, peerData);
}
peerState.setPeerDataMap(peerDataMap);
@@ -718,6 +779,7 @@
(new Integer(peerState.getMaxClientCnxns())).toString());
Map<Long, QuorumPeerData> peerDataMap = peerState.getPeerDataMap();
+ logger.debug("peerDataMap.size() = "+peerDataMap.size());
for(QuorumPeerData peerData : peerDataMap.values()) {
long peerId = peerData.getPeerId();
InetSocketAddress pAddr = peerData.getPeerAddress();
@@ -728,6 +790,7 @@
String serverKey = "server."+peerId;
String serverVal = peerAddr.getHostAddress()
+":"+peerPort+":"+electionPort;
+ logger.debug("serverKey="+serverKey+", serverVal="+serverVal);
configProps.setProperty(serverKey, serverVal);
}
return configProps;
@@ -885,10 +948,10 @@
//clientPort
//for zookeeper 3.2.1
- this.peerState.setClientPort(zConfig.getClientPort());
+// this.peerState.setClientPort(zConfig.getClientPort());
//for zookeeper 3.3.0+
-// this.peerState.setClientPort
-// (zConfig.getClientPortAddress().getPort());
+ this.peerState.setClientPort
+ (zConfig.getClientPortAddress().getPort());
//dataDir
this.peerState.setDataDir(zConfig.getDataDir());
@@ -977,6 +1040,7 @@
if( peerIdFound ) break;
}
}
+
this.peerState.setPeerDataMap(peerDataMap);
this.peerState.setNQuorumPeers(peerDataMap.size());
}
@@ -984,13 +1048,17 @@
} else {//retrieve from jini config
logger.log(Level.DEBUG, "INITIAL START: "
- +"[use jini config]");
+ +"[use deployment config]");
//clientPort
+ Integer defaultClientPort =
+ ConfigDeployUtil.getInt
+ ("federation.zookeeperClientPort");
Integer zClientPort =
(Integer)config.getEntry(COMPONENT_NAME,
"zookeeperClientPort",
- Integer.class, 2181);
+ Integer.class,
+ defaultClientPort);
if(zClientPort == null) {
throw new ConfigurationException
("null zookeeperClientPort");
@@ -998,7 +1066,9 @@
this.peerState.setClientPort(zClientPort);
//dataDir
- String defaultDataDir = "data";
+ String defaultDataDir =
+ ConfigDeployUtil.getString
+ ("federation.zookeeperDataDir");
String zDataDir = persistBaseStr + F_SEP
+ (String)config.getEntry(COMPONENT_NAME,
"zookeeperDataDir",
@@ -1011,7 +1081,9 @@
this.peerState.setDataDir(zDataDir);
//dataLogDir
- String defaultDataLogDir = "data.log";
+ String defaultDataLogDir =
+ ConfigDeployUtil.getString
+ ("federation.zookeeperDataLogDir");
String zDataLogDir = persistBaseStr + F_SEP
+ (String)config.getEntry
(COMPONENT_NAME,
@@ -1025,10 +1097,14 @@
this.peerState.setDataLogDir(zDataLogDir);
//tickTime
+ Integer defaultTickTime =
+ ConfigDeployUtil.getInt
+ ("federation.zookeepeTickTime");
Integer zTickTime =
(Integer)config.getEntry(COMPONENT_NAME,
"zookeeperTickTime",
- Integer.class, 2000);
+ Integer.class,
+ defaultTickTime);
if(zTickTime == null) {
throw new ConfigurationException
("null zookeeperTickTime");
@@ -1036,6 +1112,9 @@
this.peerState.setTickTime(zTickTime);
//initLimit
+ Integer defaultInitLimit =
+ ConfigDeployUtil.getInt
+ ("federation.zookeeperInitLimit");
Integer zInitLimit =
(Integer)config.getEntry(COMPONENT_NAME,
"zookeeperInitLimit",
@@ -1047,10 +1126,14 @@
this.peerState.setInitLimit(zInitLimit);
//syncLimit
+ Integer defaultSyncLimit =
+ ConfigDeployUtil.getInt
+ ("federation.zookeeperSyncLimit");
Integer zSyncLimit =
(Integer)config.getEntry(COMPONENT_NAME,
"zookeeperSyncLimit",
- Integer.class, 2);
+ Integer.class,
+ defaultSyncLimit);
if(zSyncLimit == null) {
throw new ConfigurationException
("null zookeeperSyncLimit");
@@ -1058,10 +1141,14 @@
this.peerState.setSyncLimit(zSyncLimit);
//electionAlg
+ Integer defaultElectionAlg =
+ ConfigDeployUtil.getInt
+ ("federation.zookeeperElectionAlg");
Integer zElectionAlg =
(Integer)config.getEntry(COMPONENT_NAME,
"zookeeperElectionAlg",
- Integer.class, 3);
+ Integer.class,
+ defaultElectionAlg);
if(zElectionAlg == null) {
throw new ConfigurationException
("null zookeeperElectionAlg");
@@ -1069,10 +1156,14 @@
this.peerState.setElectionAlg(zElectionAlg);
//maxClientCnxns
+ Integer defaultMaxClientCnxns =
+ ConfigDeployUtil.getInt
+ ("federation.zookeeperMaxClientCnxns");
Integer zMaxClientCnxns =
(Integer)config.getEntry(COMPONENT_NAME,
"zookeeperMaxClientCnxns",
- Integer.class, 10);
+ Integer.class,
+ defaultMaxClientCnxns);
if(zMaxClientCnxns == null) {
throw new ConfigurationException
("null zookeeperMaxClientCnxns");
@@ -1080,10 +1171,11 @@
this.peerState.setMaxClientCnxns(zMaxClientCnxns);
// Because this is the first time this service is started,
- // and because the config is retrieved from a jini config,
- // there is no knowledge (yet) of the other zookeeper
- // servers, other than the total number of peers that
- // are expected to make up the ensemble. If the ensemble
+ // and because the config is retrieved from a deployment
+ // specific configuration, there is no knowledge (yet)
+ // of the other zookeeper servers, other than the total
+ // number of peers that are expected to make up the
+ // ensemble (zookeeperEnsembleSize). If the ensemble
// will consist of only this service peer, then the
// peerConfigMap can be created and populated with this
// service's server information. But if the ensemble will
@@ -1092,21 +1184,31 @@
// servers are discovered and a leader is elected.
//zookeeperNetwork (peerAddress)
- String zookeeperNetwork = NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
+ String zookeeperNetwork =
+ NicUtil.getIpAddress
+ ("default.nic",
+ ConfigDeployUtil.getString
+ ("node.serviceNetwork"),
+ false);
if(zookeeperNetwork == null) {
throw new ConfigurationException
("null zookeeperNetwork");
}
InetAddress peerAddress =
- NicUtil.getInetAddress(zookeeperNetwork, 0, null, true);
+ NicUtil.getInetAddress
+ (zookeeperNetwork, 0, null, true);
this.peerState.setPeerAddress(peerAddress);
//peerPort
+ Integer defaultPeerPort =
+ ConfigDeployUtil.getInt
+ ("federation.zookeeperPeerPort");
Integer peerPort =
(Integer)config.getEntry(COMPONENT_NAME,
"zookeeperPeerPort",
- Integer.class, 2888);
+ Integer.class,
+ defaultPeerPort);
if(peerPort == null) {
throw new ConfigurationException
("null zookeeperPeerPort");
@@ -1114,6 +1216,9 @@
this.peerState.setPeerPort(peerPort);
//electionPort
+ Integer defaultElectionPort =
+ ConfigDeployUtil.getInt
+ ("federation.zookeeperElectionPort");
Integer electionPort =
(Integer)config.getEntry(COMPONENT_NAME,
"zookeeperElectionPort",
@@ -1125,10 +1230,14 @@
this.peerState.setElectionPort(electionPort);
//nQuorumPeers
+ Integer defaultEnsembleSize =
+ ConfigDeployUtil.getInt
+ ("federation.zookeeperEnsembleSize");
Integer nQuorumPeers =
(Integer)config.getEntry(COMPONENT_NAME,
- "nQuorumPeers",
- Integer.class, 1);
+ "zookeeperEnsembleSize",
+ Integer.class,
+ defaultEnsembleSize);
if(nQuorumPeers == null) {
throw new ConfigurationException
("null nQuorumPeers");
@@ -1140,7 +1249,18 @@
this.peerState.setNQuorumPeers(nQuorumPeers);
if(nQuorumPeers > 1) {
- this.peerState.setPeerId(0L);//0 - no peers discovered
+
+ // nQuorumPeers > 1 means that the ensemble is
+ // configured to not be standalone; in which case,
+ // this service's peer id is initially set to 0
+ // to indicate to this service's init method that
+ // the initQuorumPeerData method must be invoked
+ // to dicover the other peers so in the ensemble
+ // so that each peer's id (myid) can be set to a
+ // unique value between 1 and nQuorumPeers
+
+ this.peerState.setPeerId(0L);
+
} else {//nQuorumPeers == 1, populate peerConfigMap
long peerId = 1L;
this.peerState.setPeerId(peerId);
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/config/quorum.config
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/config/quorum.config 2010-11-19 18:12:50 UTC (rev 3970)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/config/quorum.config 2010-11-22 17:42:16 UTC (rev 3971)
@@ -58,14 +58,7 @@
( new String[] { System.getProperty("app.home", "${user.dir}"),
"${/}var${/}state${/}quorumState" } );
- zookeeperDataDir = "data";
- zookeeperDataLogDir = "data.log";
- zookeeperClientPort = new Integer(2181);
- zookeeperTickTime = new Integer(2000);
- zookeeperInitLimit = new Integer(5);
- zookeeperSyncLimit = new Integer(2);
- zookeeperElectionAlg = new Integer(3);//0 = udp, 3 = tcp
- zookeeperMaxClientCnxns = new Integer(10);
+ peerDiscoveryPeriod = 300000L;// wait 5 minutes for peer discovery
// If standard zookeeper config is specified,
// it will override jini config; for example,
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/ConfigDeployUtil.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/ConfigDeployUtil.java 2010-11-19 18:12:50 UTC (rev 3970)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/ConfigDeployUtil.java 2010-11-22 17:42:16 UTC (rev 3971)
@@ -400,10 +400,10 @@
throws ConfigurationException
{
String validValuesStr =
- (String) getDeploymentProperties().get(parameter + STRINGVALS);
+ (String) getDeploymentProperties().get(parameter + STRINGVALS);
if (validValuesStr != null) {
- String[] validValues = validValuesStr.split(",");
+ String[] validValues = (validValuesStr.trim()).split(",");
if (!Arrays.asList(validValues).contains(value)) {
throw new ConfigurationException
("invalid string parameter ["+parameter+"] in "
@@ -417,11 +417,11 @@
throws ConfigurationException
{
String validValuesStr =
- (String)(getDeploymentProperties().get(parameter + STRINGVALS));
+ (String)(getDeploymentProperties().get(parameter + STRINGVALS));
String[] values = value.split(",");
if (validValuesStr != null) {
- String[] validValues = validValuesStr.split(",");
+ String[] validValues = (validValuesStr.trim()).split(",");
List validValuesList = Arrays.asList(validValues);
for (int i=0; i<values.length; i++) {
if (!validValuesList.contains(values[i])) {
@@ -434,6 +434,39 @@
return values;
}
+//NOTE: there seems to be a problem with NumberFormat.parse; in particular,
+// with parsing strings representing integers. During testing
+// there were numerous occasions where, when parsing a string value
+// from the properties (from either default-deploy.properties or
+// from deploy.properties), either a NumberFormatException or
+// a ParseException would occur; even when the value being parsed
+// was valid. These exceptions, which occurred randomly, typically
+// would indicate that value being parsed was the empty string ("")
+// for the case of a NumberFormatException, or the resulting parsed
+// max (or min) value did not satisfy the desired criteria. For
+// example, when a strvalue of "2181" was input, and the maximum
+// value retrieved from the properties was "65353", upon parsing the
+// string "65353", the value 1024 was returned; thus, because 1024
+// is less than 2181, a ParseException was thrown. In other cases,
+// although a NumberFormatException was thrown by the call to
+// NumberFormat.parse because that method interpretted the string
+// "65353" as the empty string.
+//
+// Upon doing a search of the various bug databases and related
+// user reports, there seems to be some indication that the
+// parse method may at some point invoke the indexOf method on
+// the string that is being parsed. Thus, the problem being
+// described here may be related to JDK bug described at the url,
+//
+// http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6935535
+//
+// (Or it may be related to the locale being used by the tests that
+// encountered this issue?)
+//
+// As a work around, the validateInt and validateLong methods each
+// parse the given string using Integer.parseInt and Long.parseLong
+// respectively; at least until this issue is resolved.
+
private static int validateInt(String parameter, String strvalue)
throws ConfigurationException
{
@@ -445,33 +478,42 @@
int value = str2int(strvalue);
if (maxString != null) {
+/* see note ************************************************
try {
- int max = numberFormat.parse(maxString).intValue();
- if (value > max) {
- throw new ConfigurationException
- ("parameter ["+parameter+"] "
- +"exceeds maximum ["+max+"]");
- }
+ int max = numberFormat.parse(maxString.trim()).intValue();
} catch (ParseException e) {
throw new NumberFormatException
("invalid maximum integer for parameter: "
+parameter);
}
+************************************************************* */
+int max = Integer.parseInt(maxString.trim());
+ if (value > max) {
+ throw new ConfigurationException
+ ("parameter ["+parameter+"] exceeds maximum "
+ +"["+strvalue+" > "+max
+ +" (maxString="+maxString+")]");
+ }
}
if (minString != null) {
+/* see note ************************************************
try {
- int min = numberFormat.parse(minString).intValue();
- if (value < min) {
- throw new ConfigurationException
- ("parameter ["+parameter+"] "
- +"is less than maximum ["+min+"]");
- }
+ int min = numberFormat.parse(minString.trim()).intValue();
} catch (ParseException e) {
throw new NumberFormatException
("invalid minimum integer for parameter: "
+parameter);
}
+************************************************************* */
+int min = Integer.parseInt(minString.trim());
+ if (value < min) {
+ throw new ConfigurationException
+ ("parameter ["+parameter+"] is less than minimum "
+ +"["+strvalue+" < "+min
+ +" (minString="+minString+")]");
+ }
}
+
return value;
}
@@ -486,32 +528,40 @@
long value = str2long(strvalue);
if (maxString != null) {
+/* see note ************************************************
try {
- long max = numberFormat.parse(maxString).longValue();
- if (value > max) {
- throw new ConfigurationException
- ("parameter ["+parameter+"] "
- +"exceeds maximum ["+max+"]");
- }
+ long max = numberFormat.parse(maxString.trim()).longValue();
} catch (ParseException e) {
throw new NumberFormatException
("invalid maximum long for parameter: "
+parameter);
}
+************************************************************* */
+long max = Long.parseLong(maxString.trim());
+ if (value > max) {
+ throw new ConfigurationException
+ ("parameter ["+parameter+"] exceeds maximum "
+ +"["+strvalue+" > "+max
+ +" (maxString="+maxString+")]");
+ }
}
if (minString != null) {
+/* see note ************************************************
try {
- long min = numberFormat.parse(minString).longValue();
- if (value < min) {
- throw new ConfigurationException
- ("parameter ["+parameter+"] "
- +"is less than manimum ["+min+"]");
- }
+ long min = numberFormat.parse(minString.trim()).longValue();
} catch (ParseException e) {
throw new NumberFormatException
("invalid minimum long for parameter: "
+parameter);
}
+************************************************************* */
+long min = Long.parseLong(minString.trim());
+ if (value < min) {
+ throw new ConfigurationException
+ ("parameter ["+parameter+"] is less than minimum "
+ +"["+strvalue+" < "+min
+ +" (minString="+minString+")]");
+ }
}
return value;
}
@@ -603,7 +653,7 @@
//BTM - when the new version str2long was used. Thus, at
//BTM - least temporarily, until the problem can be
//BTM - diagnosed and fixed, the old versions of those
-//BTM - methods are still be used below.
+//BTM - methods are still being used below.
//BTM private static int str2int(String argx) {
//BTM Number n = null;
//BTM try {
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/default-deploy.properties
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/default-deploy.properties 2010-11-19 18:12:50 UTC (rev 3970)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/default-deploy.properties 2010-11-22 17:42:16 UTC (rev 3971)
@@ -16,12 +16,72 @@
federation.minNodes.max=1000
federation.minNodes.type=long
-federation.quorumSize.description=Number of service instances required for a quorum
-federation.quorumSize.default=1
-federation.quorumSize.min=1
-federation.quorumSize.max=11
-federation.quorumSize.type=long
+federation.zookeeperEnsembleSize.description=Number of quorum servers making up the federation's ZooKeeper ensemble
+federation.zookeeperEnsembleSize.default=1
+federation.zookeeperEnsembleSize.min=1
+federation.zookeeperEnsembleSize.max=11
+federation.zookeeperEnsembleSize.type=int
+federation.zookeeperClientPort.description=Port on which each quorum server should listen for connection requests from clients
+federation.zookeeperClientPort.default=2181
+federation.zookeeperClientPort.min=1024
+federation.zookeeperClientPort.max=65353
+federation.zookeeperClientPort.type=int
+
+federation.zookeeperDataDir.description=Path to the directory where each quorum server should persist its in-memory database snapshots, as well as its transaction logs, unless the federation.zookeeperDataLogDir parameter is set to a different value
+federation.zookeeperDataDir.default=data
+federation.zookeeperDataDir.type=string
+
+federation.zookeeperDataLogDir.description=Path to the directory where each quorum server should persist its transaction logs
+federation.zookeeperDataLogDir.default=data
+federation.zookeeperDataLogDir.type=string
+
+federation.zookeepeTickTime.description=Length of the basic time unit (in milliseconds) used by each quorum server to regulate the heartbeats and timeouts enforced by each server
+federation.zookeepeTickTime.default=2000
+federation.zookeepeTickTime.min=1
+federation.zookeepeTickTime.max=2147483647
+federation.zookeepeTickTime.type=int
+
+federation.zookeeperInitLimit.description=Amount of time (number of ticks) each quorum server should allow for followers to connect and sync with the leader in the federation's ensemble; which should be increased as needed, when the amount of data being managed by the ensemble is large
+federation.zookeeperInitLimit.default=5
+federation.zookeeperInitLimit.min=1
+federation.zookeeperInitLimit.max=2147483647
+federation.zookeeperInitLimit.type=int
+
+federation.zookeeperSyncLimit.description=Amount of time (number of ticks) each quorum server should allow for followers to sync with the leader in the federation's ensemble; where if a given follower falls too far behind the leader, the follower is dropped
+federation.zookeeperSyncLimit.default=2
+federation.zookeeperSyncLimit.min=1
+federation.zookeeperSyncLimit.max=2147483647
+federation.zookeeperSyncLimit.type=int
+
+federation.zookeeperElectionAlgorithm.description=Leader election algorithm the federation's ensemble should use when electing a leader; where a value of 0 corresponds to the original UDP-based version, 1 corresponds to the non-authenticated UDP-based version of fast leader election, 2 corresponds to the authenticated UDP-based version of fast leader election, and 3 corresponds to the TCP-based version of leader election (note that only 0 and 3 are currently supported)
+federation.zookeeperElectionAlg.default=3
+federation.zookeeperElectionAlg.min=0
+federation.zookeeperElectionAlg.max=3
+federation.zookeeperElectionAlg.type=int
+
+federation.zookeeperMaxClientCnxns.description=Maximum number of concurrent connections (at the socket level) that a single client, identified by IP address, is allowed to make to a single member of the federation's ensemble; where 0 corresponds to no limit
+federation.zookeeperMaxClientCnxns.default=0
+federation.zookeeperMaxClientCnxns.min=0
+federation.zookeeperMaxClientCnxns.max=2147483647
+federation.zookeeperMaxClientCnxns.type=int
+
+federation.zookeeperPeerPort.description=Port on which quorum servers acting as followers in the federation's ensemble request a connection to the leader in the ensemble
+federation.zookeeperPeerPort.default=2888
+federation.zookeeperPeerPort.min=1024
+federation.zookeeperPeerPort.max=65353
+federation.zookeeperPeerPort.type=int
+
+federation.zookeeperElectionPort.description=Port on which connections are requested by each quorum server when participating in leader election
+federation.zookeeperElectionPort.default=3888
+federation.zookeeperElectionPort.min=1024
+federation.zookeeperElectionPort.max=65353
+federation.zookeeperElectionPort.type=int
+
+federation.zookeeperJmxLog4j.description=When true, each peer in the federation's ensemble registers MBeans that allow one to manage log4j settings using JMX
+federation.zookeeperJmxLog4j.type=boolean
+federation.zookeeperJmxLog4j.default=false
+
# Node properties
node.name.description=logical name assigned to the node
node.name.default=
@@ -95,4 +155,3 @@
rmi.connectTimeout.type=long
rmi.connectTimeout.default=10000
rmi.connectTimeout.min=0
-
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/deploy.properties
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/deploy.properties 2010-11-19 18:12:50 UTC (rev 3970)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/deploy.properties 2010-11-22 17:42:16 UTC (rev 3971)
@@ -3,7 +3,18 @@
# Federation properties
#federation.name=com.bigdata.group.0
#federation.minNodes=1
-#federation.quorumSize=1
+#federation.zookeeperEnsembleSize=1
+#federation.zookeeperClientPort=2181
+#federation.zookeeperDataDir=data
+#federation.zookeeperDataLogDir=data
+#federation.zookeepeTickTime=2000
+#federation.zookeeperInitLimit=5
+#federation.zookeeperSyncLimit=2
+#federation.zookeeperElectionAlg=3
+#federation.zookeeperMaxClientCnxns=0
+#federation.zookeeperPeerPort=2888
+#federation.zookeeperElectionPort=3888
+#federation.zookeeperJmxLog4j=false
# Node properties
#node.name=myHost
Added: branches/dev-btm/bigdata-jini/src/test/com/bigdata/quorum/QuorumPeerServiceTest.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/test/com/bigdata/quorum/QuorumPeerServiceTest.java (rev 0)
+++ branches/dev-btm/bigdata-jini/src/test/com/bigdata/quorum/QuorumPeerServiceTest.java 2010-11-22 17:42:16 UTC (rev 3971)
@@ -0,0 +1,844 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package com.bigdata.quorum;
+
+// NOTE: remove commented out references to org.junit and annotations
+// when/if the junit infrastructure is upgraded to a version that
+// supports those constructs.
+
+import static junit.framework.Assert.*;
+
+//import static org.junit.Assert.*;
+//import org.junit.After;
+//import org.junit.BeforeClass;
+//import org.junit.Test;
+
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
+
+import com.bigdata.service.QuorumPeerService;
+import com.bigdata.util.Util;
+import com.bigdata.util.config.NicUtil;
+import com.bigdata.util.config.ConfigDeployUtil;
+import com.bigdata.util.config.ConfigurationUtil;
+import com.bigdata.util.config.LogUtil;
+
+import com.sun.jini.admin.DestroyAdmin;
+import com.sun.jini.start.NonActivatableServiceDescriptor;
+import com.sun.jini.start.NonActivatableServiceDescriptor.Created;
+import net.jini.admin.Administrable;
+import net.jini.config.AbstractConfiguration;
+import net.jini.config.ConfigurationException;
+import net.jini.core.discovery.LookupLocator;
+import net.jini.core.lookup.ServiceItem;
+import net.jini.core.lookup.ServiceRegistrar;
+import net.jini.core.lookup.ServiceTemplate;
+import net.jini.discovery.DiscoveryManagement;
+import net.jini.discovery.DiscoveryGroupManagement;
+import net.jini.discovery.DiscoveryLocatorManagement;
+import net.jini.discovery.DiscoveryListener;
+import net.jini.discovery.DiscoveryEvent;
+import net.jini.discovery.LookupDiscoveryManager;
+import net.jini.lookup.LookupCache;
+import net.jini.lookup.ServiceDiscoveryEvent;
+import net.jini.lookup.ServiceDiscoveryListener;
+import net.jini.lookup.ServiceDiscoveryManager;
+import net.jini.security.BasicProxyPreparer;
+import net.jini.security.ProxyPreparer;
+
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+import org.apache.zookeeper.ZooKeeper;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.rmi.RMISecurityManager;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+
+/*
+ * Tests the QuorumPeerService smart proxy implementation that wraps the
+ * ZooKeeper QuorumPeerMain server. Tests include starting and stopping
+ * multiple instances of the service, discovering the service as a Jini
+ * service, and verification of various client interactions with the
+ * service.
+ *
+ * NOTE: The intent of this test class is to start a single ensemble
+ * of N QuorumPeerService instances once, when this class is
+ * instantiated, allowing all test methods of this class to interact
+ * with only that one ensemble, and then shutdown the ensemble
+ * only after all tests have been run (or this class exits on
+ * failure); rather than start and stop a new ensemble with each
+ * test method executed. With more recent versions of junit, this
+ * is possible using annotations such as @BeforeClass and @After.
+ * Unfortunately, the current version of junit that is being used
+ * does not support such annotations. Additionally, the current
+ * design of the test infrastructure expects that this test class
+ * sub-classes org.junit.TestCase, which generally requires that
+ * constructors be provided with this class.
+ *
+ * In order to achieve the desired intent described above, while
+ * adhering to the requirements imposed by the current test
+ * framework, this class provide a method (beforeClassSetup) that
+ * is invoked once (from within the constructor), prior to the
+ * execution of any of the test methods; and a method that is
+ * invoked only after all the test methods of this class have been
+ * invoked (afterClassTeardown).
+ *
+ * When/if the junit infrastructure used by this test framework is
+ * ever upgraded to a version that supports annotations, and this test
+ * class is changed so that it no longer has to extend TestCase, then
+ * the appropriate changes should be made to this class to exploit
+ * the features of that new version; as indicated in the documentation
+ * below.
+ */
+public class QuorumPeerServiceTest extends TestCase {
+
+ private static String pSep = System.getProperty("path.separator");
+ private static String fSep = System.getProperty("file.separator");
+ private static String tmpDir = System.getProperty("java.io.tmpdir");
+ private static String userDir = System.getProperty("user.dir");
+ private static String policy =
+ System.getProperty("java.security.policy");
+ private static String log4jJar = System.getProperty("log4j.jar");
+ private static String jskPlatformJar =
+ System.getProperty("jsk-platform.jar");
+ private static String jskLibJar = System.getProperty("jsk-lib.jar");
+ private static String zookeeperJar = System.getProperty("zookeeper.jar");
+ private static String federationName =
+ System.getProperty("federation.name");
+ private static String bigdataRoot = System.getProperty("app.home");
+ private static String stateBase = tmpDir+fSep+"state";
+ private static String codebasePortStr = System.getProperty
+ ("codebase.port","23333");
+ private static int codebasePort = Integer.parseInt(codebasePortStr);
+
+ private static Logger logger;
+
+ // for starting smart proxy implementation of ZooKeeper quorum server
+ private static String thisHost;
+ private static String jskCodebase;
+ private static String groups;//for overriding the configured groups
+
+ private static int[] quorumClientPort = {2180, 2181, 2182};
+ private static int[] quorumPeerPort = {2887, 2888, 2889};
+ private static int[] quorumElectionPort = {3887, 3888, 3889};
+ private static String quorumServerCodebase;
+ private static String quorumCodebase;
+ private static String quorumClasspath = jskPlatformJar+pSep
+ +jskLibJar+pSep
+ +zookeeperJar+pSep
+ +log4jJar;
+ private static String quorumImplName = "com.bigdata.quorum.ServiceImpl";
+ private static String quorumConfig = bigdataRoot+fSep
+ +"dist"+fSep
+ +"bigdata"+fSep
+ +"var"+fSep
+ +"config"+fSep
+ +"jini"+fSep
+ +"quorum.config";
+ private static String[] quorumStateDir =
+ {
+ stateBase+fSep+"quorumState"+"."+quorumClientPort[0]
+ +"."+quorumPeerPort[0]
+ +"."+quorumElectionPort[0],
+ stateBase+fSep+"quorumState"+"."+quorumClientPort[1]
+ +"."+quorumPeerPort[1]
+ +"."+quorumElectionPort[1],
+ stateBase+fSep+"quorumState"+"."+quorumClientPort[2]
+ +"."+quorumPeerPort[2]
+ +"."+quorumElectionPort[2]
+ };
+ private static String[] quorumPersistenceOverride =
+ {
+ "com.bigdata.quorum.persistenceDirectory=new String("
+ +"\""+quorumStateDir[0]+"\")",
+ "com.bigdata.quorum.persistenceDirectory=new String("
+ +"\""+quorumStateDir[1]+"\")",
+ "com.bigdata.quorum.persistenceDirectory=new String("
+ +"\""+quorumStateDir[2]+"\")"
+ };
+ private static int nQuorumServicesExpected =
+ quorumPersistenceOverride.length;
+ private static boolean quorumsAlreadyStarted = false;
+ private static HashSet<QuorumPeerService> quorumSet =
+ new HashSet<QuorumPeerService>();
+
+ private static String[] groupsToDiscover = new String[] {"qaQuorumGroup"};
+ private static LookupLocator[] locsToDiscover = new LookupLocator[0];
+ private static DiscoveryManagement ldm;
+ protected static ServiceDiscoveryManager sdm;
+ private static CacheListener cacheListener;
+ private static LookupCache quorumCache;
+
+ private static ExecutorService serviceStarterTaskExecutor =
+ Executors.newFixedThreadPool(quorumStateDir.length);
+
+ private static boolean setupOnceAlready = false;
+ private static boolean exceptionInSetup = false;
+ private static boolean lastTest = false;
+
+ // When using ServiceStarter to start the desired service instances,
+ // need to hold references to each service instance that is created
+ // to prevent distributed garbage collection on each service ref. The
+ // map below is used to hold those references; where the map's key
+ // is the persistence directory path for the corresponding service
+ // reference.
+ private static Map<String, Created> refMap =
+ new ConcurrentHashMap<String, Created>();
+
+ private String testName;
+ private boolean testPassed;
+
+ // NOTE: remove constructors and when/if the junit infrastructure
+ // is upgraded to a version that supports annotations and this test
+ // is changed so that it no longer has to extend TestCase.
+ public QuorumPeerServiceTest() throws Exception {
+ beforeClassSetup();
+ }
+
+ public QuorumPeerServiceTest(String name) throws Exception {
+ super(name);
+ beforeClassSetup();
+ }
+
+ // Test framework methods ------------------------------------------------
+
+ // Intended to be run before any test methods are executed. This method
+ // starts all services and creates any resources whose life cycles
+ // are intended to span all tests; rather than being set up and torn down
+ // from test to test.
+ //
+ // NOTE: use the @BeforeClass annotation when/if the junit framework is
+ // upgraded to a version that supports annotations.
+// @BeforeClass public static void beforeClassSetup() {
+ public static synchronized void beforeClassSetup() throws Exception {
+ if (setupOnceAlready) return;
+ setupOnceAlready = true;
+ try {
+ String logConfigFile = userDir+fSep+"bigdata"+fSep+"src"+fSep
+ +"resources"+fSep+"logging"+fSep
+ +"log4j.properties";
+ System.setProperty("log4j.configuration", logConfigFile);
+ logger = LogUtil.getLog4jLogger
+ ( (QuorumPeerServiceTest.class).getName() );
+ logger.debug("\n\n-- beforeClassSetup ENTER ----------\n");
+
+ // Setup both lookup & service discovery, plus groups, codebase ...
+ setupDiscovery();
+
+ // Start the services making up the ensemble
+
+ String ensembleSize = "com.bigdata.quorum.zookeeperEnsembleSize="
+ +"new Integer("+quorumStateDir.length+")";
+ long discoveryPeriod = 10L*1000L;
+ String discoveryPeriodStr =
+ "com.bigdata.quorum.peerDiscoveryPeriod="+discoveryPeriod;
+ String jmxLog4j =
+ "com.bigdata.quorum.zookeeperJmxLog4j=new Boolean("+false+")";
+ for (int i=0; i<quorumStateDir.length; i++) {
+ String clientPort =
+ "com.bigdata.quorum.zookeeperClientPort="
+ +"new Integer("+quorumClientPort[i]+")";
+ String peerPort =
+ "com.bigdata.quorum.zookeeperPeerPort="
+ +"new Integer("+quorumPeerPort[i]+")";
+ String electionPort =
+ "com.bigdata.quorum.zookeeperElectionPort="
+ +"new Integer("+quorumElectionPort[i]+")";
+ String joinGroups =
+ "com.bigdata.quorum.groupsToJoin=new String[] "
+ +groups;
+
+ serviceStarterTaskExecutor.execute
+ ( new ServiceStarterTask(quorumStateDir[i],
+ ensembleSize,
+ clientPort,
+ peerPort,
+ electionPort,
+ quorumPersistenceOverride[i],
+ joinGroups,
+ discoveryPeriodStr,
+ jmxLog4j) );
+ }
+
+ // Give the services time to start & rendezvous with each other
+ int nWait = 10;
+ for (int i=0; i<nWait; i++) {
+ if (refMap.size() == quorumStateDir.length) break;
+ Util.delayMS(1L*1000L);
+ }
+ if (refMap.size() != quorumStateDir.length) {
+ throw new Exception("did not start all expected services "
+ +"[expected="+quorumStateDir.length
+ +", actual="+refMap.size()+"]");
+ }
+ } catch(Exception e) {
+ exceptionInSetup = true;
+ throw e;
+ }
+ logger.debug("\n\n-- beforeClassSetup EXIT ----------\n");
+ }
+
+ // Intended to be run after all test methods have completed executing.
+ // This method terminates all services and cleans up resources whose
+ // life cycles span all tests; rather than being set up and torn down
+ // from test to test.
+ //
+ // NOTE: use the @AfterClass annotation when/if the junit framework is
+ // upgraded to a version that supports annotations.
+// @AfterClass public void afterClassTearDown() {
+ public void afterClassTearDown() {
+ logger.debug("\n\n-- afterClassTearDown ENTER ----------\n");
+ if(sdm != null) {
+ try {
+ sdm.terminate();
+ logger.log(Level.INFO, "terminated sdm");
+ } catch(Throwable t) { }
+ }
+
+ if(ldm != null) {
+ try {
+ ldm.terminate();
+ logger.log(Level.INFO, "terminated ldm");
+ } catch(Throwable t) { }
+ }
+
+ stopAllQuorums();
+
+ // Clean up persistence directories. Note that a delay is injected
+ // before attempting to delete the persistence directories. This
+ // is done to allow ZooKeeper to fully complete whatever clean up
+ // processing it performs. Through trial and error it has been
+ // found that without such a delay, although this test class
+ // seems to complete its processing successfully, the next test
+ // class that is run often times exits before executing any
+ // test methods; without any indication of what caused the
+ // premature exit. Thus, until it can be determined what is
+ // actually causing this issue, and how to address it, the delay
+ // performed below will be used to allow ZooKeeper to clean
+ // up appropriately.
+
+ Util.delayMS(1L*1000L);//delay 1 second
+ for (int i=0; i<quorumStateDir.length; i++) {
+ String persistenceDirectory = quorumStateDir[i];
+...
[truncated message content] |
|
From: <btm...@us...> - 2010-12-06 22:09:50
|
Revision: 3997
http://bigdata.svn.sourceforge.net/bigdata/?rev=3997&view=rev
Author: btmurphy
Date: 2010-12-06 22:09:42 +0000 (Mon, 06 Dec 2010)
Log Message:
-----------
[branch dev-btm]: CHECKPOINT - changes to allow smart proxy wrapper for zookeeper to be started by ServicesManagerService; also changes to allow TestServiceStarter test to start smart proxy zookeeper [more fixes to junit tests needed]
Modified Paths:
--------------
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServicesManagerStartupTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServicesManagerConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ZookeeperServerConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/ZookeeperProcessHelper.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/JiniServicesHelper.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/AbstractFedZooTestCase.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/TestServiceStarter.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/zookeeper/AbstractZooTestCase.java
branches/dev-btm/src/resources/config/bigdataCluster.config
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServicesManagerStartupTask.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServicesManagerStartupTask.java 2010-12-06 21:29:19 UTC (rev 3996)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServicesManagerStartupTask.java 2010-12-06 22:09:42 UTC (rev 3997)
@@ -141,7 +141,12 @@
if (serviceConfig instanceof ZookeeperServerConfiguration) {
System.out.println("\n---- ServicesManagerStartupTask.doStartup: startZookeeperService() ----");
- startZookeeperService(config);
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - BEGIN
+//BTM - PRE_ZOOKEEPER_SMART_PROXY startZookeeperService(config);
+ startZookeeperService
+ ( ((ZookeeperServerConfiguration)serviceConfig).classType,
+ config );
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - END
}
@@ -227,23 +232,38 @@
*
* @return <code>true</code> if an instance was started successfully.
*/
- protected boolean startZookeeperService(final Configuration config)
- throws ConfigurationException, IOException {
-
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - BEGIN
+//BTM - PRE_ZOOKEEPER_SMART_PROXY protected boolean startZookeeperService(final Configuration config)
+//BTM - PRE_ZOOKEEPER_SMART_PROXY throws ConfigurationException, IOException {
+//BTM - PRE_ZOOKEEPER_SMART_PROXY
+//BTM - PRE_ZOOKEEPER_SMART_PROXY try {
+//BTM - PRE_ZOOKEEPER_SMART_PROXY
+//BTM - PRE_ZOOKEEPER_SMART_PROXY return ZookeeperProcessHelper.startZookeeper(config, service) > 0;
+//BTM - PRE_ZOOKEEPER_SMART_PROXY
+//BTM - PRE_ZOOKEEPER_SMART_PROXY } catch (Throwable t) {
+//BTM - PRE_ZOOKEEPER_SMART_PROXY
+//BTM - PRE_ZOOKEEPER_SMART_PROXY log.error(
+//BTM - PRE_ZOOKEEPER_SMART_PROXY "Could not start zookeeper service: " + t, t);
+//BTM - PRE_ZOOKEEPER_SMART_PROXY
+//BTM - PRE_ZOOKEEPER_SMART_PROXY return false;
+//BTM - PRE_ZOOKEEPER_SMART_PROXY
+//BTM - PRE_ZOOKEEPER_SMART_PROXY }
+//BTM - PRE_ZOOKEEPER_SMART_PROXY
+//BTM - PRE_ZOOKEEPER_SMART_PROXY }
+//BTM - PRE_ZOOKEEPER_SMART_PROXY
+ protected boolean startZookeeperService
+ (Class classType, Configuration config)
+ throws ConfigurationException, IOException
+ {
try {
-
- return ZookeeperProcessHelper.startZookeeper(config, service) > 0;
-
+ return ZookeeperProcessHelper.startZookeeper
+ (classType, config, service) > 0;
} catch (Throwable t) {
-
- log.error(
- "Could not start zookeeper service: " + t, t);
-
+ log.error("Could not start zookeeper service: " + t, t);
return false;
-
}
-
}
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - END
/**
* If necessary, start the jini core services on this host.
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServicesManagerConfiguration.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServicesManagerConfiguration.java 2010-12-06 21:29:19 UTC (rev 3996)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServicesManagerConfiguration.java 2010-12-06 22:09:42 UTC (rev 3997)
@@ -225,7 +225,7 @@
} else if (a.equals(QuorumPeerMain.class.getName())) {
System.out.println("\n*** ServicesManagerConfiguration.getConfigurations: QuorumPeerMain BEGIN");
- v.add(new ZookeeperServerConfiguration(config));
+ v.add(new ZookeeperServerConfiguration(QuorumPeerMain.class, config));
System.out.println("*** ServicesManagerConfiguration.getConfigurations: QuorumPeerMain END\n");
} else if (a.equals(TransactionServer.class.getName())) {
@@ -261,6 +261,12 @@
//BTM - BEGIN: smart proxy impls ------------------------------------------------------------
+ } else if (a.equals(com.bigdata.quorum.ServiceImpl.class.getName())) {
+
+System.out.println("\n*** ServicesManagerConfiguration.getConfigurations: com.bigdata.quorum.ServiceImpl BEGIN");
+ v.add(new ZookeeperServerConfiguration(com.bigdata.quorum.ServiceImpl.class, config));
+System.out.println("*** ServicesManagerConfiguration.getConfigurations: com.bigdata.quorum.ServiceImpl END\n");
+
} else if (a.equals(com.bigdata.transaction.ServiceImpl.class.getName())) {//transaction service
System.out.println("\n*** ServicesManagerConfiguration.getConfigurations: com.bigdata.transaction.ServiceImpl BEGIN");
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ZookeeperServerConfiguration.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ZookeeperServerConfiguration.java 2010-12-06 21:29:19 UTC (rev 3996)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ZookeeperServerConfiguration.java 2010-12-06 22:09:42 UTC (rev 3997)
@@ -257,15 +257,32 @@
}
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - BEGIN
+ public Class classType;
+ private Configuration jiniConfig;
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - END
+
/**
* @param className
* @param config
* @throws ConfigurationException
*/
- public ZookeeperServerConfiguration(final Configuration config)
- throws ConfigurationException {
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - BEGIN
+//BTM - PRE_ZOOKEEPER_SMART_PROXY public ZookeeperServerConfiguration(final Configuration config)
+//BTM - PRE_ZOOKEEPER_SMART_PROXY throws ConfigurationException {
+//BTM - PRE_ZOOKEEPER_SMART_PROXY
+//BTM - PRE_ZOOKEEPER_SMART_PROXY super(QuorumPeerMain.class.getName(), config);
+ public ZookeeperServerConfiguration(Class classType,
+ Configuration config)
+ throws ConfigurationException
+ {
- super(QuorumPeerMain.class.getName(), config);
+ super(classType.getName(), config);
+ this.classType = classType;
+ this.jiniConfig = config;
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - END
+//BTM
+System.out.println("*** ZookeeperServerConfiguration: constructor ***");
servers = (String) config.getEntry(Options.NAMESPACE, Options.SERVERS,
String.class);
@@ -455,6 +472,7 @@
public ZookeeperServiceStarter newServiceStarter(IServiceListener listener,
ZookeeperServerEntry entry) {
+System.out.println("*** ZookeeperServerConfiguration ---> newServiceStarter ***");
return new ZookeeperServiceStarter(listener, entry);
}
@@ -511,6 +529,7 @@
ZookeeperServerEntry entry) {
super(listener);
+System.out.println("*** ZookeeperServerConfiguration.ZookeeperServiceStarter: constructor - BEGIN ***");
if (entry == null)
throw new IllegalArgumentException();
@@ -524,6 +543,7 @@
*/
dataDir = new File(ZookeeperServerConfiguration.this.dataDir,
Integer.toString(entry.id)).getAbsoluteFile();
+System.out.println("*** ZookeeperServerConfiguration.ZookeeperServiceStarter: dataDir = "+dataDir);
if (log.isInfoEnabled())
log.info(Options.DATA_DIR + "=" + dataDir);
@@ -536,6 +556,7 @@
*/
dataLogDir = new File(ZookeeperServerConfiguration.this.dataLogDir,
Integer.toString(entry.id)).getAbsoluteFile();
+System.out.println("*** ZookeeperServerConfiguration.ZookeeperServiceStarter: dataLogDir = "+dataLogDir);
if (log.isInfoEnabled())
log.info(Options.DATA_LOG_DIR + "=" + dataLogDir);
@@ -544,9 +565,12 @@
configFile = new File(dataDir,
ZookeeperServerConfiguration.this.configFile)
.getAbsoluteFile();
+System.out.println("*** ZookeeperServerConfiguration.ZookeeperServiceStarter: generated configFile = "+configFile);
// the server id is written on this file.
myidFile = new File(dataDir, "myid");
+System.out.println("*** ZookeeperServerConfiguration.ZookeeperServiceStarter: myidFile = "+myidFile);
+System.out.println("*** ZookeeperServerConfiguration.ZookeeperServiceStarter: constructor - END ***");
}
@@ -618,7 +642,14 @@
/*
* Write the zookeeper server configuration file.
*/
- writeZookeeperConfigFile();
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - BEGIN
+//BTM - PRE_ZOOKEEPER_SMART_PROXY writeZookeeperConfigFile();
+ if ( (Options.NAMESPACE).equals(classType.getName()) ) {
+ writeZookeeperConfigFile();
+ } else {
+ writeConfigFile();// Jini config file
+ }
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - END
/*
* Start the server.
@@ -682,7 +713,9 @@
// the configuration file.
cmds.add(configFile.toString());
-
+//BTM - FOR_ZOOKEEPER_SMART_PROXY - BEGIN
+ cmds.add("com.bigdata.quorum."+getGroups("groupsToJoin"));
+//BTM - FOR_ZOOKEEPER_SMART_PROXY - END
}
@Override
@@ -859,6 +892,113 @@
}
+//BTM - FOR_ZOOKEEPER_SMART_PROXY - BEGIN ------------------------------------
+ protected void writeConfigFile() throws IOException {
+
+ // 1. generate the contents to be written to the file
+ final String contents;
+ {
+ final StringWriter out = new StringWriter();
+ writeConfigFile(out);
+ out.flush();
+ contents = out.toString();
+ }
+
+ // 2. write the contents generated above to the file.
+ {
+ final Writer out2 =
+ new OutputStreamWriter
+ (new BufferedOutputStream
+ (new FileOutputStream(configFile)));
+ try {
+ out2.write(contents);
+ out2.flush();
+ } finally {
+ out2.close();
+ }
+ }
+ }
+
+ protected void writeConfigFile(final Writer out) throws IOException {
+ // 1. Comments
+ writeComments(out);
+ out.write("\n");
+
+ // 2. import statements
+ for (String i : getImports()) {// write import statements
+ out.write("import " + i + ";\n");
+ }
+ out.write("\n");
+
+ // 3. Open component name
+ out.write("\n\n" + classType.getPackage().getName() + " {\n");
+ {
+ out.write(" "+Options.DATA_DIR+" = new File("+"\""+dataDir.toString()+"\""+");\n");
+ out.write(" "+Options.DATA_LOG_DIR+" = new File("+"\""+dataLogDir.toString()+"\""+");\n");
+ out.write(" "+Options.CLIENT_PORT+" = "+clientPort+";\n");
+ out.write(" "+Options.SERVERS+" = "+"\""+servers+"\""+";\n");
+ for(Map.Entry<String, String> entry : other.entrySet()) {
+ String key = entry.getKey();
+ String val = entry.getValue();
+ out.write(" "+key+" = "+val+";\n");
+ }
+
+ out.write(" "+getGroups("groupsToJoin")+";\n");
+ out.write(" locsToJoin=new LookupLocator[]{ };\n");
+ out.write(" static discoveryManager = new LookupDiscoveryManager(groupsToJoin,locsToJoin,null,this);\n");
+ }// END BLOCK - 4. Close component name
+ out.write("}\n");
+ }
+
+ protected void writeComments(final Writer out) throws IOException {
+ out.write("// className=" + className + "\n");
+ out.write("// date=" + new java.util.Date() + "\n");
+ }
+
+ public String[] getImports() {
+ return new String[]
+ { "java.net.NetworkInterface",
+ "com.sun.jini.config.ConfigUtil",
+ "net.jini.constraint.BasicMethodConstraints",
+ "net.jini.core.constraint.ConnectionRelativeTime",
+ "net.jini.core.constraint.InvocationConstraints",
+ "net.jini.jeri.BasicILFactory",
+ "net.jini.jeri.BasicJeriExporter",
+ "net.jini.jeri.tcp.TcpServerEndpoint",
+ "net.jini.core.discovery.LookupLocator",
+ "net.jini.discovery.LookupDiscoveryManager",
+ "com.bigdata.util.config.NicUtil",
+ "com.bigdata.util.config.ConfigDeployUtil"
+ };
+ }
+
+ protected String getGroups(String entryName) {
+ String fedname = null;
+ String zrootname = null;//in some test configs
+ String defaultGroupName = "UNKNOWN";
+ try {
+ fedname = (String) jiniConfig.getEntry
+ ("bigdata", "fedname",
+ String.class, "fedname-unset");
+ zrootname = (String) jiniConfig.getEntry
+ ("bigdata", "zrootname",
+ String.class, null);
+ } catch(Exception e) {//swallow
+ }
+ if (fedname == null) fedname = defaultGroupName;
+
+ StringBuffer strBuf =
+ new StringBuffer(entryName+"=new String[]{");
+ strBuf.append( com.bigdata.jini.util.ConfigMath.q(fedname) );
+ if (zrootname != null) {
+ strBuf.append
+ (","+com.bigdata.jini.util.ConfigMath.q(zrootname));
+ }
+ strBuf.append("}");
+
+ return strBuf.toString();
+ }
+//BTM - FOR_ZOOKEEPER_SMART_PROXY - END --------------------------------------
}
/**
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/ZookeeperProcessHelper.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/ZookeeperProcessHelper.java 2010-12-06 21:29:19 UTC (rev 3996)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/ZookeeperProcessHelper.java 2010-12-06 22:09:42 UTC (rev 3997)
@@ -147,13 +147,23 @@
*
* @see ZookeeperServerConfiguration#newServiceStarter(IServiceListener, ZookeeperServerEntry)
*/
- static public int startZookeeper(final Configuration config,
- final IServiceListener listener) throws ConfigurationException,
- IOException {
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - BEGIN
+//BTM - PRE_ZOOKEEPER_SMART_PROXY static public int startZookeeper(final Configuration config,
+//BTM - PRE_ZOOKEEPER_SMART_PROXY final IServiceListener listener) throws ConfigurationException,
+//BTM - PRE_ZOOKEEPER_SMART_PROXY IOException {
+//BTM - PRE_ZOOKEEPER_SMART_PROXY
+//BTM - PRE_ZOOKEEPER_SMART_PROXY final ZookeeperServerConfiguration serverConfig = new ZookeeperServerConfiguration(
+//BTM - PRE_ZOOKEEPER_SMART_PROXY config);
+ static public int startZookeeper
+ (final Class classType,
+ final Configuration config,
+ final IServiceListener listener)
+ throws ConfigurationException, IOException
+ {
+ final ZookeeperServerConfiguration serverConfig =
+ new ZookeeperServerConfiguration(classType, config);
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - END
- final ZookeeperServerConfiguration serverConfig = new ZookeeperServerConfiguration(
- config);
-
//BTM log.warn("\n------------ ZookeeperProcessHelper.startZookeeper: [localhost="+thisInetAddr.getHostName()+", clientPort="+serverConfig.clientPort+"]\n");
//BTM com.bigdata.util.Util.printStr("TestBigdata.debug","\n------------ ZookeeperProcessHelper.startZookeeper: [localhost="+thisInetAddr.getHostName()+", clientPort="+serverConfig.clientPort+"]\n");
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/ServiceImpl.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/ServiceImpl.java 2010-12-06 21:29:19 UTC (rev 3996)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/ServiceImpl.java 2010-12-06 22:09:42 UTC (rev 3997)
@@ -98,6 +98,7 @@
import java.net.SocketException;
import java.net.UnknownHostException;
import java.rmi.RemoteException;
+import java.rmi.RMISecurityManager;
import java.rmi.server.ExportException;
import java.util.ArrayList;
import java.util.Collections;
@@ -117,11 +118,20 @@
/**
* Backend (admin) zookeeper based implementation of the quorum peer service.
+ *
+ * Note: this class is currently declared public rather than the preferred
+ * package protected scope. This is so that the JiniServicesHelper
+ * utility can instantiate this class in the tests that are currently
+ * implemented to interact directly with the service's backend;
+ * as opposed to starting the service with the ServiceStarter and
+ * then interacting with the service through the discovered service
+ * frontend.
*/
+public
class ServiceImpl implements PrivateInterface {
- private static Logger logger = LogUtil.getLog4jLogger
- ( (ServiceImpl.class).getName() ) ;
+ private static Logger logger =
+ LogUtil.getLog4jLogger(COMPONENT_NAME);
private static String shutdownStr;
private static String killStr;
@@ -267,6 +277,9 @@
//Initialize the service from the config
private void init(String[] args) throws Exception {
+ if(System.getSecurityManager() == null) {
+ System.setSecurityManager(new RMISecurityManager());
+ }
config = ConfigurationProvider.getInstance
( args,
(this.getClass()).getClassLoader() );
@@ -1388,4 +1401,116 @@
}
}
}
+
+ /**
+ * This main() method is provided because it may be desirable (for
+ * testing or other reasons) to be able to start this service using
+ * a command line that is either manually entered in a command window
+ * or supplied to the java ProcessBuilder class for execution.
+ * <p>
+ * The mechanism that currently employs the ProcessBuilder class to
+ * execute a dynamically generated command line will be referred to
+ * as the 'ServiceConfiguration mechanism', which involves the use of
+ * the following ServiceConfiguration class hierarchy,
+ * <p>
+ * <ul>
+ * <li> ZookeeperConfiguration
+ * <li> JavaServiceConfiguration
+ * <li> ServiceConfiguration
+ * </ul>
+ * </p>
+ * The ServicesConfiguration mechanism may involve the use of the
+ * ServicesManagerService directly to execute this service, or it may
+ * involve the use of the junit framework to start this service. In
+ * either case, a command line is constructed from information that is
+ * specified at each of the various ServiceConfiguration levels, and
+ * is ultimately executed in a ProcessBuilder instance (in the
+ * ProcessHelper class).
+ * <p>
+ * In order for this method to know whether or not the
+ * ServiceConfiguration mechanism is being used to start the service,
+ * this method must be told that the ServiceConfiguration mechanism is
+ * being used. This is done by setting the system property named
+ * <code>usingServiceConfiguration</code> to any non-null value.
+ * <p>
+ * When the ServiceConfiguration mechanism is <i>not</i> used to start
+ * this service, this method assumes that the element at index 0
+ * of the args array references the path to the jini configuration
+ * file that will be input by this method to this service's constructor.
+ * On the other hand, when the ServiceConfiguration mechanism <i>is</i>
+ * used to start this service, the service's configuration is handled
+ * differently, as described below.
+ * <p>
+ * When using the ServiceConfiguration mechanism, in addition to
+ * generating a command line to start the service, although an initial,
+ * pre-constructed jini configuration file is supplied (to the
+ * ServicesManagerService or the test framework, for example), a
+ * second jini configuration file is generated <i>on the fly</i> as
+ * well. When generating that new configuration file, a subset of the
+ * components and entries specified in the initial jini configuration
+ * are retrieved and placed in the new configuration being generated.
+ * It is that second, newly-generated configuration file that is input
+ * to this method through the args elements at index 0.
+ * <p>
+ * When the ServiceConfiguration mechanism is used to invoke this
+ * method, this method makes a number of assumptions. One assumption
+ * is that there is a component with name equal to the value,
+ * "org.apache.zookeeper.server.quorum.QuorumPeerMain", as well as
+ * either a component itself, or entries corresponding to a component,
+ * with name equal to the fully qualified name of this class (or both).
+ * Another assumption is that an entry named 'args' is associated with
+ * that component. The 'args' entry is assumed to be a <code>String</code>
+ * array in which one of the elments is specified to be a system
+ * property named 'config' whose value is equal to the path and
+ * filename of yet a third jini configuration file; that is, something
+ * of the form, "-Dconfig=<path-to-another-jini-config>". It is this
+ * third jini configuration file that the service will ultimately use
+ * to initialize itself when the ServiceConfiguration mechanism is
+ * being used to start the service. In that case then, this method
+ * will retrieve the path to the third jini configuration file from
+ * the configuration file supplied to this method in the args array
+ * at index 0, and then replace the element at index 0 with that
+ * path; so that when the service is instantiated (using this class'
+ * constructor), that third configuration file is made available to
+ * the service instance.
+ * <p>
+ * Note that, unlike the other service implementations, this service
+ * always generates its own service id the very first time it is
+ * started, persists that service id, and retrieves and reuses it
+ * on restarts.
+ * <p>
+ * Note that once an instance of this service implementation class
+ * has been created, that instance is stored in the <code>thisImpl</code>
+ * field to prevent the instance from being garbage collected until
+ * the service is actually shutdown.
+ */
+
+ private static ServiceImpl thisImpl;
+
+ public static void main(String[] args) {
+ logger.debug("[main]: appHome="+System.getProperty("appHome"));
+ try {
+ // If the system property with name "config" is set, then
+ // use the value of that property to override the value
+ // input in the first element of the args array
+ ArrayList<String> argsList = new ArrayList<String>();
+ int begIndx = 0;
+ String configFile = System.getProperty("config");
+ if(configFile != null) {
+ // Replace args[0] with config file location
+ argsList.add(configFile);
+ begIndx = 1;
+ }
+ for(int i=begIndx; i<args.length; i++) {
+ argsList.add(args[i]);
+ }
+ logger.debug("[main]: instantiating service [new ServiceImpl]");
+ thisImpl = new ServiceImpl
+ ( argsList.toArray(new String[argsList.size()]),
+ new com.bigdata.service.jini.FakeLifeCycle() );
+ } catch(Throwable t) {
+ logger.log(Level.WARN,
+ "failed to start callable executor service", t);
+ }
+ }
}
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/JiniServicesHelper.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/JiniServicesHelper.java 2010-12-06 21:29:19 UTC (rev 3996)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/JiniServicesHelper.java 2010-12-06 22:09:42 UTC (rev 3997)
@@ -555,8 +555,14 @@
// start zookeeper (a server instance).
//BTM log.warn("\n---------------- JiniServicesHelper.innerStart >>> START ZOOKEEPER\n");
//BTM com.bigdata.util.Util.printStr("TestBigdata.debug","\n---------------- JiniServicesHelper.innerStart >>> START ZOOKEEPER\n");
- final int nstarted = ZookeeperProcessHelper.startZookeeper(
- config, serviceListener);
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - BEGIN
+//BTM - PRE_ZOOKEEPER_SMART_PROXY final int nstarted = ZookeeperProcessHelper.startZookeeper(
+//BTM - PRE_ZOOKEEPER_SMART_PROXY config, serviceListener);
+ final int nstarted =
+ ZookeeperProcessHelper.startZookeeper
+ (com.bigdata.quorum.ServiceImpl.class, //BTM - was QuorumPeerMain.class
+ config, serviceListener);
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - END
//BTM log.warn("\n---------------- JiniServicesHelper.innerStart >>> START ZOOKEEPER - DONE\n");
//BTM com.bigdata.util.Util.printStr("TestBigdata.debug","\n---------------- JiniServicesHelper.innerStart >>> START ZOOKEEPER - DONE\n");
Modified: branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/AbstractFedZooTestCase.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/AbstractFedZooTestCase.java 2010-12-06 21:29:19 UTC (rev 3996)
+++ branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/AbstractFedZooTestCase.java 2010-12-06 22:09:42 UTC (rev 3997)
@@ -50,6 +50,16 @@
import com.bigdata.service.jini.JiniClient;
import com.bigdata.service.jini.JiniFederation;
+//BTM - FOR_ZOOKEEPER_SMART_PROXY - BEGIN
+import com.bigdata.service.QuorumPeerService;
+import com.sun.jini.admin.DestroyAdmin;
+import net.jini.admin.Administrable;
+import net.jini.core.lookup.ServiceItem;
+import net.jini.core.lookup.ServiceTemplate;
+import net.jini.discovery.DiscoveryGroupManagement;
+import net.jini.lookup.ServiceDiscoveryManager;
+//BTM - FOR_ZOOKEEPER_SMART_PROXY - END
+
/**
* Abstract base class for unit tests requiring a running zookeeper and a
* running federation as configured from a test resource.
@@ -143,7 +153,11 @@
//BTM - FOR_CLIENT_SERVICE - END
// if necessary, start zookeeper (a server instance).
- ZookeeperProcessHelper.startZookeeper(config, listener);
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - BEGIN
+//BTM - PRE_ZOOKEEPER_SMART_PROXY ZookeeperProcessHelper.startZookeeper(config, listener);
+ ZookeeperProcessHelper.startZookeeper(com.bigdata.quorum.ServiceImpl.class, config, listener);
+//ZookeeperProcessHelper.startZookeeper(org.apache.zookeeper.server.quorum.QuorumPeerMain.class, config, listener);
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - END
/*
* FIXME We need to start a jini lookup service for groups = {fedname}
@@ -171,6 +185,29 @@
System.err.println(getName() + ": tearing down zrootname=" + zrootname);
+//BTM - FOR_ZOOKEEPER_SMART_PROXY - BEGIN
+ // Graceful shutdown of QuorumPeerService
+ ServiceDiscoveryManager sdm = fed.getServiceDiscoveryManager();
+ DiscoveryGroupManagement ldm =
+ (DiscoveryGroupManagement)(sdm.getDiscoveryManager());
+ Class[] quorumServiceType =
+ new Class[] {com.bigdata.service.QuorumPeerService.class};
+ ServiceTemplate quorumServiceTmpl =
+ new ServiceTemplate(null, quorumServiceType, null);
+ ServiceItem[] items =
+ sdm.lookup(quorumServiceTmpl, Integer.MAX_VALUE, null);
+
+ for (int i=0; i<items.length; i++) {
+ QuorumPeerService zk = (QuorumPeerService)(items[i].service);
+ try {
+ Object admin = ((Administrable)zk).getAdmin();
+ ((DestroyAdmin)admin).destroy();
+ } catch(Exception e) {
+ log.warn("failure on zookeeper destroy ["+zk+"]", e);
+ }
+ }
+//BTM - FOR_ZOOKEEPER_SMART_PROXY - END
+
// destroy any processes started by this test suite.
for (ProcessHelper t : listener.running) {
Modified: branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/TestServiceStarter.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/TestServiceStarter.java 2010-12-06 21:29:19 UTC (rev 3996)
+++ branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/TestServiceStarter.java 2010-12-06 22:09:42 UTC (rev 3997)
@@ -176,7 +176,8 @@
final ServiceItem serviceItem;
IService proxy = null;
Service smartProxy = null;
- final String physicalServiceZPath;
+//BTM (12/06/2010) final String physicalServiceZPath;
+String physicalServiceZPath = null;
{
final List<String> children = zookeeper.getChildren(
@@ -194,14 +195,52 @@
* Note: You could explicitly build the correct zpath using the
* serviceUUID obtained from the service proxy.
*/
- physicalServiceZPath = logicalServiceZPath + "/"
- + children.get(0);
-
- // get the serviceUUID from the physicalServiceZNode's data.
- final UUID serviceUUID = (UUID) SerializerUtil
- .deserialize(zookeeper.getData(physicalServiceZPath,
- false/* watch */, new Stat()));
-
+//BTM - BEGIN (12/06/2010) ----------------------------------------------
+//BTM The note above appears to be no longer true. That is,
+//BTM at some point, the tests or infrastructure were changed
+//BTM in such a way that there are now 2 children instead of
+//BTM the 1 child the note above says is expected. Currently,
+//BTM the children are masterElection and physicalServices,
+//BTM but the code below that retrieves the serviceUUID from
+//BTM zookeeper must retrieve it from the physicalServices
+//BTM children, not the masterElection child. The original
+//BTM code invoked children.get(0), assuming either there
+//BTM was only 1 child or assuming that the physicalServices
+//BTM child would always be placed at index 0 of the
+//BTM children list above zookeeper.getChildren() was called.
+//BTM Unfortunately, this is not alway the case. And when/if
+//BTM an attempt is made to deserialize the serviceUUID using
+//BTM the masterElection child, an EOF exception is encountered
+//BTM because serviceUUID information was never stored under
+//BTM the masterElection znode. To address this then, the
+//BTM code below was changed to loop through the children
+//BTM list, catching the exception and exiting when a
+//BTM a valid serviceUUID is successfully deserialized.
+//BTM
+//BTM physicalServiceZPath = logicalServiceZPath + "/"
+//BTM + children.get(0);
+//BTM // get the serviceUUID from the physicalServiceZNode's data.
+//BTM final UUID serviceUUID = (UUID) SerializerUtil
+//BTM .deserialize(zookeeper.getData(physicalServiceZPath,
+//BTM false/* watch */, new Stat()));
+//BTM
+ UUID serviceUUID = null;
+ for (String child : children) {
+ physicalServiceZPath = logicalServiceZPath+"/"+child;
+ try {
+ serviceUUID =
+ (UUID) SerializerUtil.deserialize
+ ( zookeeper.getData(physicalServiceZPath,
+ false,//watch
+ new Stat()) );
+ break;
+ } catch(Exception e) {//swallow
+ }
+ }
+ assertTrue( "failed to deserialize serviceUUID "
+ +"[children="+children+"]",
+ (serviceUUID != null) );
+//BTM - END --------------------------------------------------------------
serviceItem = discoverService(serviceUUID);
// verify that the service item is registered with jini.
Modified: branches/dev-btm/bigdata-jini/src/test/com/bigdata/zookeeper/AbstractZooTestCase.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/test/com/bigdata/zookeeper/AbstractZooTestCase.java 2010-12-06 21:29:19 UTC (rev 3996)
+++ branches/dev-btm/bigdata-jini/src/test/com/bigdata/zookeeper/AbstractZooTestCase.java 2010-12-06 22:09:42 UTC (rev 3997)
@@ -153,12 +153,16 @@
if (log.isInfoEnabled())
log.info(getName());
-
+
// find ports that are not in use.
clientPort = getPort(2181/* suggestedPort */);
final int peerPort = getPort(2888/* suggestedPort */);
final int leaderPort = getPort(3888/* suggestedPort */);
- final String servers = "1=localhost:" + peerPort + ":" + leaderPort;
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - BEGIN
+//BTM - PRE_ZOOKEEPER_SMART_PROXY final String servers = "1=localhost:" + peerPort + ":" + leaderPort;
+ String hostname = com.bigdata.util.config.NicUtil.getIpAddress("default.nic", "default", true);
+ final String servers = "1="+hostname+":" + peerPort + ":" + leaderPort;
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - END
// create a temporary file for zookeeper's state.
dataDir = File.createTempFile("test", ".zoo");
@@ -200,9 +204,14 @@
this.sessionTimeout = tickTime * 2;
// if necessary, start zookeeper (a server instance).
- ZookeeperProcessHelper.startZookeeper(config, listener);
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - BEGIN
+//BTM - PRE_ZOOKEEPER_SMART_PROXY ZookeeperProcessHelper.startZookeeper(config, listener);
+//BTM - PRE_ZOOKEEPER_SMART_PROXY
+//BTM - PRE_ZOOKEEPER_SMART_PROXY zookeeperAccessor = new ZooKeeperAccessor("localhost:" + clientPort, sessionTimeout);
- zookeeperAccessor = new ZooKeeperAccessor("localhost:" + clientPort, sessionTimeout);
+ zookeeperAccessor = new ZooKeeperAccessor(hostname+":" + clientPort, sessionTimeout);
+ ZookeeperProcessHelper.startZookeeper(com.bigdata.quorum.ServiceImpl.class, config, listener);
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - END
zookeeper = zookeeperAccessor.getZookeeper();
Modified: branches/dev-btm/src/resources/config/bigdataCluster.config
===================================================================
--- branches/dev-btm/src/resources/config/bigdataCluster.config 2010-12-06 21:29:19 UTC (rev 3996)
+++ branches/dev-btm/src/resources/config/bigdataCluster.config 2010-12-06 22:09:42 UTC (rev 3997)
@@ -1371,6 +1371,21 @@
properties = new NV[] {
};
}
+
+com.bigdata.quorum.ServiceImpl {
+
+ args = new String[]{
+ "-Xmx200m",
+
+ "-Djava.util.logging.config.file=@NAS@/dist/bigdata/var/config/logging/quorum-logging.properties",
+ "-Dlog4j.configuration=@NAS@/dist/bigdata/var/config/logging/quorum-logging.properties",
+ "-Dlog4j.primary.configuration=@NAS@/dist/bigdata/var/config/logging/quorum-logging.properties",
+ "-DusingServiceConfiguration=true",
+ "-Dbigdata.logDir=@NAS@/dist/bigdata/var/log",
+ "-DappHome=@APP_HOME@",
+ "-Dconfig=@NAS@/dist/bigdata/var/config/jini/quorum.config"
+ };
+}
//BTM - END
/**
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
×
Want the latest updates on software, tech news, and AI?
Get latest updates about software, tech news, and AI from SourceForge directly in your inbox once a month.
Thanks for helping keep SourceForge clean.
X