|
From: <btm...@us...> - 2010-07-23 12:19:21
|
Revision: 3269
http://bigdata.svn.sourceforge.net/bigdata/?rev=3269&view=rev
Author: btmurphy
Date: 2010-07-23 12:19:14 +0000 (Fri, 23 Jul 2010)
Log Message:
-----------
[branch dev-btm]: CHECKPOINT - smart proxy work, still incomplete and including debug println's and BTM markers, but checkpointed for syncing with trunk changes
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractCounterSet.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/LocalDataServiceFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/util/config/NicUtil.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/StressTestConcurrent.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/AbstractServicesManagerService.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ManageLogicalServiceTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/MonitorCreatePhysicalServiceLocksTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/RestartPersistentServices.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServiceConfigurationZNodeMonitorTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServicesManagerServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServicesManagerStartupTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/AbstractHostConstraint.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/BigdataServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/JavaServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/JiniCoreServicesConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/JiniServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/LoadBalancerConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ManagedServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServicesManagerConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ZookeeperServerEntry.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/JiniCoreServicesProcessHelper.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/JiniServiceProcessHelper.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/ProcessHelper.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/ZookeeperProcessHelper.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/EmbeddedLoadBalancer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/ServiceProxy.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/config/logging.properties
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/AbstractServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/JiniFederation.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/lookup/LoadBalancerClient.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/JiniServicesHelper.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/LookupStarter.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/config/lookup.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/Util.java
branches/dev-btm/bigdata-jini/src/resources/config/bigdataStandaloneTesting.config
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/config/TestZookeeperServerEntry.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/testReggie.config
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/testStartJini.config
branches/dev-btm/bigdata-jini/src/test/com/bigdata/zookeeper/testzoo.config
branches/dev-btm/build.properties
branches/dev-btm/build.xml
branches/dev-btm/src/resources/config/bigdataCluster.config
branches/dev-btm/src/resources/config/jini/reggie.config
branches/dev-btm/src/resources/config/log4j.properties
branches/dev-btm/src/resources/config/logging.properties
Added Paths:
-----------
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/EntryUtil.java
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractCounterSet.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractCounterSet.java 2010-07-23 11:04:00 UTC (rev 3268)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractCounterSet.java 2010-07-23 12:19:14 UTC (rev 3269)
@@ -42,6 +42,8 @@
*/
public abstract class AbstractCounterSet implements ICounterSet {
+public static org.apache.log4j.Logger logger = com.bigdata.util.config.LogUtil.getLog4jLogger((AbstractCounterSet.class).getName());
+
protected final String name;
protected AbstractCounterSet parent;
@@ -198,11 +200,39 @@
if( path.contains("//")) {
+if (logger.isEnabledFor(org.apache.log4j.Level.WARN)) {
+ logger.warn("***** AbstractCounterSet.getPath: CONTAINS SLASH-SLASH: path INPUT = "+path);
+ logger.warn("***** AbstractCounterSet.getPath: path.length = "+path.length());
+ byte[] pathBytes = path.getBytes();
+ StringBuffer strBuf = new StringBuffer();
+ if( (pathBytes[0] < 32) || (pathBytes[0] > 126) ) {
+ strBuf.append("X");
+ }else{
+ strBuf.append( new String(new byte[] {pathBytes[0]}) );
+ }
+ for(int i=1;i<pathBytes.length; i++) {
+ if( (pathBytes[i] < 32) || (pathBytes[i] > 126) ) {
+ strBuf.append("X");
+ }else{
+ strBuf.append( new String(new byte[] {pathBytes[i]}) );
+ }
+ }
+ logger.warn("***** AbstractCounterSet.getPath: CONTAINS SLASH-SLASH: path CONVERTED = "+strBuf.toString());
+}
+int slashSlashIndex = path.indexOf("//");
+if(slashSlashIndex >= 0) {//contains "//"
+ logger.warn("***** AbstractCounterSet.getPath: INDEX OF SLASH-SLASH = "+slashSlashIndex);
+}else{//does not contain "//"
+ logger.warn("***** AbstractCounterSet.getPath: !!!! path CONTAINS SLASH-SLASH, but indexOf = "+slashSlashIndex+" !!!!");
+}
+
+if(slashSlashIndex >= 0) {//throw exception only if BOTH path.contains AND path.indexOf say that the given path contains "//"
/*
* Empty path names are not allowed.
*/
throw new IllegalArgumentException(path);
+}//BTM
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-07-23 11:04:00 UTC (rev 3268)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-07-23 12:19:14 UTC (rev 3269)
@@ -1388,7 +1388,7 @@
// Note: This _is_ a local method call.
final UUID serviceUUID = fed.getServiceUUID();
-
+System.out.println("\n>>>>> AbstractFederation.reportPerformanceCounters: serviceUUID = "+serviceUUID);
// Will be null until assigned by the service registrar.
if (serviceUUID == null) {
@@ -1403,12 +1403,14 @@
final LoadBalancer loadBalancerService = fed.getLoadBalancerService();
if (loadBalancerService == null) {
+System.out.println(">>>>> AbstractFederation.reportPerformanceCounters: loadBalancerService = NULL");
log.warn("Could not discover load balancer service.");
return;
}
+System.out.println(">>>>> AbstractFederation.reportPerformanceCounters: loadBalancerService = "+loadBalancerService);
/*
* @todo this is probably worth compressing as there will be a lot
@@ -1419,9 +1421,12 @@
final ByteArrayOutputStream baos = new ByteArrayOutputStream(
Bytes.kilobyte32 * 2);
+System.out.println(">>>>> AbstractFederation.reportPerformanceCounters: CALLING fed.getCounterSet() ...");
fed.getCounterSet().asXML(baos, "UTF-8", null/* filter */);
+System.out.println(">>>>> AbstractFederation.reportPerformanceCounters: CALLING loadBalancer.notify ...");
loadBalancerService.notify(serviceUUID, baos.toByteArray());
+System.out.println(">>>>> AbstractFederation.reportPerformanceCounters: DONE CALLING loadBalancer.notify");
if (log.isInfoEnabled())
log.info("Notified the load balancer.");
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedFederation.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedFederation.java 2010-07-23 11:04:00 UTC (rev 3268)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedFederation.java 2010-07-23 12:19:14 UTC (rev 3269)
@@ -620,6 +620,9 @@
//BTM p.setProperty(LoadBalancerService.Options.TRANSIENT, "true");
p.setProperty(EmbeddedLoadBalancerServiceImpl.Options.TRANSIENT, "true");
+p.setProperty(EmbeddedLoadBalancerServiceImpl.Options.LOG_DIR,
+ new File(EmbeddedLoadBalancerServiceImpl.Options.DEFAULT_LOG_DIR).toString());
+
} else {
// specify the data directory for the load balancer.
@@ -860,6 +863,7 @@
{
super(serviceUUID, hostname,
sdm,
+properties.getProperty(EmbeddedLoadBalancerServiceImpl.Options.LOG_DIR),
//BTM*** EmbeddedFederation.this,
dataServiceMap,//BTM*** - remove after DataService smart proxy?
properties);
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/LocalDataServiceFederation.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/LocalDataServiceFederation.java 2010-07-23 11:04:00 UTC (rev 3268)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/LocalDataServiceFederation.java 2010-07-23 12:19:14 UTC (rev 3269)
@@ -193,6 +193,7 @@
loadBalancerService = new EmbeddedLoadBalancer(UUID.randomUUID(),
hostname,
null,//SDM - replace with real SDM after conversion to smart proxy?
+p.getProperty(EmbeddedLoadBalancer.Options.LOG_DIR),
//BTM*** LocalDataServiceFederation.this,
//BTM*** remove after EmbeddedDataService is converted to smart proxy?
dataServiceMap,
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/util/config/NicUtil.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/util/config/NicUtil.java 2010-07-23 11:04:00 UTC (rev 3268)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/util/config/NicUtil.java 2010-07-23 12:19:14 UTC (rev 3269)
@@ -505,6 +505,253 @@
}
/**
+ * Special-purpose convenience method that returns a
+ * <code>String</code> value representing the ip address of
+ * the current node; where the value that is returned is
+ * determined according to following criteria:
+ * <p>
+ * <ul>
+ * <li> If a non-<code>null</code> value is input for the
+ * <code>systemPropertyName</code> parameter, then
+ * this is viewed as a declaration by the caller that
+ * that the system property with that given value
+ * should take precedence over all other means of
+ * determining the desired ip address. As such, this
+ * method determines if a system property having the
+ * given has indeed been set and, if it has, returns
+ * the ip address of the nic having that name; or
+ * <code>null</code> if there is no nic with the
+ * desired name installed on the node.
+ * <li> If a non-<code>null</code> value is input for the
+ * <code>systemPropertyName</code> parameter, but
+ * no system property with that name has been set,
+ * and <code>true</code> has been passed in for
+ * the <code>fallbackOk</code> parameter, then this
+ * method will return the IPV4 based address of the
+ * first reachable nic that is found on the node.
+ * Upon failing to find such an address, if the
+ * <code>loopbackOk</code> parameter is also
+ * <code>true</code>, then this method will return
+ * the <i>loop back</i> address of the node; otherwise
+ * <code>null</code> is returned.
+ * <li> If <code>null</code> is input for the
+ * <code>systemPropertyName</code> parameter, but
+ * a non-<code>null</code> value is input for the
+ * <code>defaultNic</code> parameter, then this
+ * method returns the ip address of the nic having
+ * that name; or <code>null</code> if there is no
+ * nic with the desired default name installed on the
+ * node.
+ * <li> If <code>null</code> is input for both the
+ * <code>systemPropertyName</code> parameter and the
+ * <code>defaultNic</code> parameter, and if the
+ * <code>fallbackOk</code> parameter is <code>true</code>,
+ * then this method will return the IPV4 based address
+ * of the first reachable nic that is found on the node.
+ * Upon failing to find such an address, if the
+ * <code>loopbackOk</code> parameter is also
+ * <code>true</code>, then this method will return
+ * the <i>loop back</i> address of the node; otherwise
+ * <code>null</code> is returned.
+ * </ul>
+ * <p>
+ * This method can be called from within a configuration
+ * as well as from within program control.
+ *
+ * @param systemPropertyName <code>String</code> value containing
+ * the name of a system property whose
+ * value is the network interface name
+ * whose ip address should be returned.
+ * May be <code>null</code>.
+ *
+ * @param defaultNic <code>String</code> value containing
+ * the name of the network interface
+ * whose ip address should be returned
+ * if <code>null</code> is input for the
+ * <code>systemPropertyName</code> parameter.
+ *
+ * @param fallbackOk if <code>true</code>, then if either
+ * no system property is set having the
+ * name referenced by the
+ * <code>systemPropertyName</code> parameter,
+ * or if <code>null</code> is input for both
+ * the <code>systemPropertyName</code>
+ * parameter and the <code>defaultNic</code>
+ * parameter, return the IPV4 based address
+ * of the first reachable network interface
+ * that can be found on the node.
+ *
+ * @param loopbackOk if <code>true</code>, and if <code>true</code>
+ * is also input for the <code>fallbackOk</code>
+ * parameter, then if this method attempts,
+ * but fails, to find a valid IPV4 fallback
+ * address, then the node's <i>loop back</i>
+ * address is returned.
+ *
+ * @return a <code>String</code> representing an ip address associated
+ * with the current node; where the value that is returned is
+ * determined according to the criteria described above.
+ */
+ public static String getIpAddress(String systemPropertyName,
+ String defaultNic,
+ boolean fallbackOk,
+ boolean loopbackOk)
+ throws SocketException, IOException
+ {
+ if(systemPropertyName != null) {//system property takes precedence
+ String nicName = System.getProperty(systemPropertyName);
+ boolean propSet = true;
+ if(nicName == null) {
+ propSet = false;
+ } else {
+ // handle ant script case where the system property
+ // may not have been set on the command line, but
+ // was still set to "${<systemPropertyName>}" using
+ // ant <sysproperty> tag
+ String rawProp = "${" + systemPropertyName + "}";
+ if( rawProp.equals(nicName) ) propSet = false;
+ }
+ if(propSet) {
+ return getIpAddress(nicName, 0, loopbackOk);
+ } else {//desired system property not set, try fallback
+ if(fallbackOk) {
+ return getDefaultIpv4Address(loopbackOk);
+ } else {
+ return null;
+ }
+ }
+ } else {//no system property name provided, try default nic
+ if(defaultNic != null) {
+ return getIpAddress(defaultNic, 0, loopbackOk);
+ } else {//no default nic provided, try fallback
+ if(fallbackOk) {
+ return getDefaultIpv4Address(loopbackOk);
+ } else {
+ return null;
+ }
+ }
+ }
+ }
+
+ public static String getIpAddress()
+ throws SocketException, IOException
+ {
+ return getIpAddress(null, null, true, true);
+ }
+
+ /**
+ * Examines each address associated with each network interface
+ * card (nic) installed on the current node, and returns the
+ * <code>String</code> value of the first such address that is
+ * determined to be both <i>reachable</i> and an address type
+ * that represents an <i>IPv4</i> address.
+ *
+ * This method will always first examine addresses that are
+ * <i>not</i> the <i>loopback</i> address (<i>local host</i>);
+ * returning a loopback adddress only if <code>true</code>
+ * is input for the <code>loopbackOk</code> parameter, and
+ * none of the non-loopback addresses satisfy this method's
+ * search criteria.
+ *
+ * If this method fails to find any address that satisfies the
+ * above criteria, then this method returns <code>null</code>.
+ *
+ * @param loopbackOk if <code>true</code>, then upon failure
+ * find an non-<i>loopback</i> address that
+ * satisfies this method's search criteria
+ * (an IPv4 type address and reachable), the
+ * first loopback address that is found to be
+ * reachable is returned.
+ *
+ * If <code>false</code> is input for this
+ * parameter, then this method will examine
+ * only those addresses that do <i>not</i>
+ * correspond to the corresponding nic's
+ * loopback address.
+ *
+ * @return a <code>String</code> value representing the first
+ * network interface address installed on the current
+ * node that is determined to be both <i>reachable</i>
+ * and an IPv4 type address; where the return value
+ * corresponds to a <i>loopback</i> address only if
+ * <code>true</code> is input for the <code>loopbackOk</code>
+ * parameter, and no non-loopback address satisfying
+ * the desired criteria can be found. If this method
+ * fails to find any address that satisfies the desired
+ * criteria, then <code>null</code> is returned.
+ *
+ * @throws SocketException if there is an error in the underlying
+ * I/O subsystem and/or protocol while retrieving the
+ * the network interfaces currently installed on the
+ * node.
+ *
+ * @throws IOException if a network error occurs while determining
+ * if a candidate return address is <i>reachable</i>.
+ */
+ public static String getDefaultIpv4Address(boolean loopbackOk)
+ throws SocketException, IOException
+ {
+ //get all nics on the current node
+ Enumeration<NetworkInterface> nics =
+ NetworkInterface.getNetworkInterfaces();
+ while( nics.hasMoreElements() ) {
+ NetworkInterface curNic = nics.nextElement();
+ List<InterfaceAddress> interfaceAddrs =
+ curNic.getInterfaceAddresses();
+ for(InterfaceAddress interfaceAddr : interfaceAddrs) {
+ InetAddress inetAddr = interfaceAddr.getAddress();
+ boolean isIpv4 = inetAddr instanceof Inet4Address;
+ boolean isLoopbackAddress = inetAddr.isLoopbackAddress();
+ if(isIpv4) {
+ if(isLoopbackAddress) continue;
+ boolean isReachable = inetAddr.isReachable(3*1000);
+ Inet4Address inet4Addr = (Inet4Address)inetAddr;
+ String retVal = inet4Addr.getHostAddress();
+
+ jiniConfigLogger.log
+ (CONFIG, "default IPv4 address: "+retVal);
+ utilLogger.log
+ (Level.TRACE, "default IPv4 address: "+retVal);
+ return retVal;
+ }
+ }
+ }
+
+ if(!loopbackOk) return null;
+
+ nics = NetworkInterface.getNetworkInterfaces();
+ while( nics.hasMoreElements() ) {
+ NetworkInterface curNic = nics.nextElement();
+ List<InterfaceAddress> interfaceAddrs =
+ curNic.getInterfaceAddresses();
+ for(InterfaceAddress interfaceAddr : interfaceAddrs) {
+ InetAddress inetAddr = interfaceAddr.getAddress();
+ boolean isIpv4 = inetAddr instanceof Inet4Address;
+ boolean isLoopbackAddress = inetAddr.isLoopbackAddress();
+ if(isIpv4) {
+ if(!isLoopbackAddress) continue;
+ boolean isReachable = inetAddr.isReachable(3*1000);
+ Inet4Address inet4Addr = (Inet4Address)inetAddr;
+ String retVal = inet4Addr.getHostAddress();
+
+ jiniConfigLogger.log
+ (CONFIG, "default IPv4 address: "+retVal);
+ utilLogger.log
+ (Level.TRACE, "default IPv4 address: "+retVal);
+ return retVal;
+ }
+ }
+ }
+ return null;
+ }
+
+ public static String getDefaultIpv4Address()
+ throws SocketException, IOException
+ {
+ return getDefaultIpv4Address(false);//localhost NOT ok
+ }
+
+ /**
* Intended for use by scripts.
*/
public static void main(String[] args) {
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/service/StressTestConcurrent.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/service/StressTestConcurrent.java 2010-07-23 11:04:00 UTC (rev 3268)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/service/StressTestConcurrent.java 2010-07-23 12:19:14 UTC (rev 3269)
@@ -791,10 +791,12 @@
throws IOException {
// explicitly set the log level for the load balancer.
- LoadBalancerService.log.setLevel(Level.INFO);
+//BTM LoadBalancerService.log.setLevel(Level.INFO);
+EmbeddedLoadBalancer.logger.setLevel(Level.INFO);
- final AbstractEmbeddedLoadBalancerService lbs = ((AbstractEmbeddedLoadBalancerService) ((EmbeddedFederation<?>) fed)
- .getLoadBalancerService());
+//BTM final AbstractEmbeddedLoadBalancerService lbs = ((AbstractEmbeddedLoadBalancerService) ((EmbeddedFederation<?>) fed)
+//BTM .getLoadBalancerService());
+final EmbeddedLoadBalancer lbs = ((EmbeddedLoadBalancer)((EmbeddedFederation)fed).getLoadBalancerService());
final ServiceScore[] fakeServiceScores = new ServiceScore[2];
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/ServiceImpl.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/ServiceImpl.java 2010-07-23 11:04:00 UTC (rev 3268)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/ServiceImpl.java 2010-07-23 12:19:14 UTC (rev 3269)
@@ -286,7 +286,7 @@
+", locators="
+Util.writeArrayElementsToString(locatorsToJoin));
- waitThread = new Util.WaitOnInterruptThread();
+ waitThread = new Util.WaitOnInterruptThread(logger);
waitThread.start();
readyState.ready();//ready to accept calls from clients
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/AbstractServicesManagerService.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/AbstractServicesManagerService.java 2010-07-23 11:04:00 UTC (rev 3268)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/AbstractServicesManagerService.java 2010-07-23 12:19:14 UTC (rev 3269)
@@ -67,6 +67,7 @@
* processes.
*/
+System.out.println("\nXXXXXXX AbstractServicesManagerService.add >>>>> KILLING process = "+service.name);
service.kill(true/* immediateShutdown */);
} catch (InterruptedException e) {
@@ -121,6 +122,7 @@
// halt our monitor tasks.
getFederation().cancelMonitoredTasks(true/* mayInterruptIfRunning */);
+System.out.println("\nXXXXXXX AbstractServicesManagerService.shutdown >>>>> KILLING CHILD PROCESSES");
// kill our children.
killChildProcesses(false/* immediateShutdown */);
@@ -146,6 +148,7 @@
// halt our monitor tasks.
getFederation().cancelMonitoredTasks(true/* mayInterruptIfRunning */);
+System.out.println("\nXXXXXXX AbstractServicesManagerService.shutdownNow >>>>> KILLING CHILD PROCESSES");
// kill our children.
killChildProcesses(true/* immediateShutdown */);
@@ -186,6 +189,7 @@
try {
+System.out.println("\nXXXXXXX AbstractServicesManagerService.killChildProcesses >>>>> calling Process.kill");
helper.kill(immediateShutdown);
} catch (Throwable t) {
@@ -301,6 +305,7 @@
/*
* Run startup.
*/
+System.out.println("\n*** AbstractServicesManagerService: setup() - ServicesManagerStartupTask().call() ***\n");
new ServicesManagerStartupTask(fed, config, true/* pushConfig */,
true/* restartServices */, this/* listener */).call();
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ManageLogicalServiceTask.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ManageLogicalServiceTask.java 2010-07-23 11:04:00 UTC (rev 3268)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ManageLogicalServiceTask.java 2010-07-23 12:19:14 UTC (rev 3269)
@@ -107,6 +107,7 @@
this.configZPath = configZPath;
this.children = children;
this.config = config;
+System.out.println("\nEEEE ManageLogicalServiceTask.constructor: configZPath="+this.configZPath);
}
@@ -178,6 +179,7 @@
*/
protected void newLogicalService() throws KeeperException, InterruptedException {
+System.out.println("\nEEEE ManageLogicalServiceTask.newLogicalService: className="+config.className);
if (INFO)
log.info("className=" + config.className);
@@ -194,31 +196,38 @@
/*
* Create zpath for the new logical service.
*/
+System.out.println("EEEE ManageLogicalServiceTask.newLogicalService: CREATING ZNODE ---> configZPath="+configZPath+", LOGICAL_SERVICE_PREFIX="+BigdataZooDefs.LOGICAL_SERVICE_PREFIX+", path="+(configZPath+"/"+BigdataZooDefs.LOGICAL_SERVICE_PREFIX));
final String logicalServiceZPath = zookeeper.create(configZPath + "/"
+ BigdataZooDefs.LOGICAL_SERVICE_PREFIX, new byte[0], acl,
CreateMode.PERSISTENT_SEQUENTIAL);
+System.out.println("EEEE ManageLogicalServiceTask.newLogicalService: CREATED ZNODE ---> logicalServiceZPath="+logicalServiceZPath);
/*
* The new znode (last path component of the new zpath).
*/
final String logicalServiceZNode = logicalServiceZPath
.substring(logicalServiceZPath.lastIndexOf('/') + ...
[truncated message content] |
|
From: <btm...@us...> - 2010-07-23 18:43:07
|
Revision: 3276
http://bigdata.svn.sourceforge.net/bigdata/?rev=3276&view=rev
Author: btmurphy
Date: 2010-07-23 18:43:00 +0000 (Fri, 23 Jul 2010)
Log Message:
-----------
merge -r3143::HEAD(3267) ~/bigdata/trunk ~/bigdata/branches/dev-btm [trunk --> branch dev-btm]
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/CollatorEnum.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/JDKSortKeyGenerator.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/GlobalRowStoreHelper.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestICUUnicodeKeyBuilder.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestJDKUnicodeKeyBuilder.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java
branches/dev-btm/bigdata/src/test/com/bigdata/cache/TestRingBuffer.java
branches/dev-btm/bigdata/src/test/com/bigdata/sparse/TestAll.java
branches/dev-btm/bigdata/src/test/com/bigdata/sparse/TestKeyEncodeDecode.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/MonitorCreatePhysicalServiceLocksTask.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/TruthMaintenance.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestNamedGraphs.java
branches/dev-btm/build.xml
branches/dev-btm/src/resources/config/bigdataCluster.config
branches/dev-btm/src/resources/config/bigdataCluster16.config
branches/dev-btm/src/resources/config/standalone/bigdataStandalone.config
Added Paths:
-----------
branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/AbstractUnicodeKeyBuilderTestCase.java
Removed Paths:
-------------
branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractUnicodeKeyBuilderTestCase.java
Property Changed:
----------------
branches/dev-btm/
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/
branches/dev-btm/dsi-utils/
branches/dev-btm/dsi-utils/src/test/
branches/dev-btm/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/src/resources/bin/config/
Property changes on: branches/dev-btm
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/bugfix-btm:2594-2779
/trunk:2575-2594,2596-2877,2882-2903,2910-3143
+ /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/bugfix-btm:2594-3237
/branches/fko:3150-3194
/trunk:2575-2594,2596-2877,2882-2903,2910-3269
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/CollatorEnum.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/CollatorEnum.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/CollatorEnum.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -10,7 +10,8 @@
/**
* The JDK bundles support for generating Unicode sort keys, but that
- * support does NOT include compressed sort keys.
+ * support does NOT include compressed sort keys and embeds <code>nul</code>
+ * bytes into its Unicode sort keys.
*/
JDK,
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -144,6 +144,7 @@
/**
* Representation includes all aspects of the {@link Serializable} state.
*/
+ @Override
public String toString() {
StringBuilder sb = new StringBuilder(getClass().getName());
@@ -270,14 +271,10 @@
/*
* Figure out which collator to use.
- *
- * Note: The default depends on whether or not the ICU library is on
- * the class path. When it is, we always default to the ICU library.
*/
collator = CollatorEnum.valueOf(getProperty(properties,
- Options.COLLATOR, (icu_avail ? CollatorEnum.ICU.toString()
- : CollatorEnum.JDK.toString())));
+ Options.COLLATOR, CollatorEnum.ICU.toString()));
// true iff the collator was _explicitly_ specified.
final boolean explicitCollatorChoice = getProperty(properties,
@@ -349,7 +346,7 @@
* Figure out the collator strength.
*/
- Object strength = null;
+ Object tmpStrength = null;
final String val = getProperty(properties, Options.STRENGTH);
@@ -357,24 +354,24 @@
try {
- strength = StrengthEnum.valueOf(val);
+ tmpStrength = StrengthEnum.valueOf(val);
} catch (RuntimeException ex) {
- strength = Integer.parseInt(val);
+ tmpStrength = Integer.parseInt(val);
}
}
if (log.isInfoEnabled())
- log.info(Options.STRENGTH + "=" + strength);
+ log.info(Options.STRENGTH + "=" + tmpStrength);
/*
* Note: MAY be null (when null, does not override the collator's
* default).
*/
- this.strength = strength;
+ this.strength = tmpStrength;
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -28,7 +28,6 @@
import java.util.Locale;
-import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import com.ibm.icu.text.Collator;
@@ -90,18 +89,6 @@
protected static final Logger log = Logger.getLogger(ICUSortKeyGenerator.class);
/**
- * True iff the {@link #log} level is INFO or less.
- */
- final protected boolean INFO = log.getEffectiveLevel().toInt() <= Level.INFO
- .toInt();
-
- /**
- * True iff the {@link #log} level is DEBUG or less.
- */
- final protected boolean DEBUG = log.getEffectiveLevel().toInt() <= Level.DEBUG
- .toInt();
-
- /**
* Used to encode unicode strings into compact byte[]s that have the same
* sort order (aka sort keys).
*/
@@ -128,7 +115,7 @@
this.locale = locale;
- if(INFO) log.info("locale="+locale);
+ if(log.isInfoEnabled()) log.info("locale="+locale);
this.collator = (RuleBasedCollator) Collator.getInstance(locale);
@@ -138,7 +125,7 @@
final int str = ((Integer) strength).intValue();
- if (INFO)
+ if (log.isInfoEnabled())
log.info("strength=" + str);
collator.setStrength(str);
@@ -147,7 +134,7 @@
StrengthEnum str = (StrengthEnum) strength;
- if (INFO)
+ if (log.isInfoEnabled())
log.info("strength=" + str);
switch (str) {
@@ -184,7 +171,7 @@
if (mode != null) {
- if(INFO) log.info("mode="+mode);
+ if(log.isInfoEnabled()) log.info("mode="+mode);
switch (mode) {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/JDKSortKeyGenerator.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/JDKSortKeyGenerator.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/JDKSortKeyGenerator.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -56,8 +56,8 @@
}
- public JDKSortKeyGenerator(Locale locale, Object strength,
- DecompositionEnum mode) {
+ public JDKSortKeyGenerator(final Locale locale, final Object strength,
+ final DecompositionEnum mode) {
if (locale == null)
throw new IllegalArgumentException();
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -180,9 +180,9 @@
* The buffer reference is used directly rather than making a
* copy of the data.
*/
- protected KeyBuilder(UnicodeSortKeyGenerator sortKeyGenerator, int len,
- byte[] buf) {
-
+ protected KeyBuilder(final UnicodeSortKeyGenerator sortKeyGenerator,
+ final int len, final byte[] buf) {
+
if (len < 0)
throw new IllegalArgumentException("len");
@@ -350,7 +350,7 @@
* The object responsible for generating sort keys from Unicode strings.
*
* The {@link UnicodeSortKeyGenerator} -or- <code>null</code> if Unicode
- * is not supported by this {@link KeyBuilder} instance.
+ * is not supported by this {@link IKeyBuilder} instance.
*/
final public UnicodeSortKeyGenerator getSortKeyGenerator() {
@@ -391,11 +391,11 @@
public KeyBuilder appendASCII(final String s) {
- int len = s.length();
+ int tmpLen = s.length();
- ensureFree(len);
+ ensureFree(tmpLen);
- for(int j=0; j<len; j++) {
+ for(int j=0; j<tmpLen; j++) {
char ch = s.charAt(j);
@@ -1356,6 +1356,11 @@
}
+ /**
+ * Create an instance for ASCII keys.
+ *
+ * @return The new instance.
+ */
public static IKeyBuilder newInstance() {
return newInstance(DEFAULT_INITIAL_CAPACITY);
@@ -1393,10 +1398,8 @@
/**
* Optional property specifies the library that will be used to generate
- * sort keys from Unicode data. The default always supports Unicode, but
- * the library choice depends on whether or not ICU library is found on
- * the classpath. When the ICU library is present, it is the default.
- * Otherwise the JDK library is the default. You may explicitly specify
+ * sort keys from Unicode data. The ICU library is the default.
+ * You may explicitly specify
* the library choice using one of the {@link CollatorEnum} values. The
* {@link CollatorEnum#ASCII} value may be used to disable Unicode
* support entirely, treating the characters as if they were ASCII. If
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -149,6 +149,7 @@
// allow client requests to finish normally.
new ShutdownHelper(threadPool, 10L/*logTimeout*/, TimeUnit.SECONDS) {
+ @Override
public void logTimeout() {
log.warn("Awaiting thread pool termination: elapsed="
@@ -170,6 +171,7 @@
new ShutdownHelper(scheduledExecutorService, 10L/* logTimeout */,
TimeUnit.SECONDS) {
+ @Override
public void logTimeout() {
log.warn("Awaiting sample service termination: elapsed="
@@ -650,12 +652,7 @@
TimeUnit.MILLISECONDS // unit
);
- addScheduledTask(//
- new StartDeferredTasksTask(),// task to run.
- 150, // initialDelay (ms)
- 2000, // delay
- TimeUnit.MILLISECONDS // unit
- );
+ getExecutorService().execute(new StartDeferredTasksTask());
// Setup locator.
resourceLocator = new DefaultResourceLocator(this,
@@ -888,12 +885,12 @@
*/
public boolean isServiceReady() {
- final AbstractClient<T> client = this.client;
+ final AbstractClient<T> thisClient = this.client;
- if (client == null)
+ if (thisClient == null)
return false;
- final IFederationDelegate<T> delegate = client.getDelegate();
+ final IFederationDelegate<T> delegate = thisClient.getDelegate();
if (delegate == null)
return false;
@@ -963,11 +960,11 @@
}
// @todo really, we should test like this everywhere.
- final AbstractClient client = this.client;
+ final AbstractClient thisClient = this.client;
- if (client != null && client.isConnected()) {
+ if (thisClient != null && thisClient.isConnected()) {
- client.getDelegate().serviceLeave(serviceUUID);
+ thisClient.getDelegate().serviceLeave(serviceUUID);
}
@@ -996,7 +993,7 @@
static private String ERR_SERVICE_NOT_READY = "Service is not ready yet.";
/**
- * This task runs periodically. Once {@link #getServiceUUID()} reports a
+ * This task runs once. Once {@link #getServiceUUID()} reports a
* non-<code>null</code> value, it will start an (optional)
* {@link AbstractStatisticsCollector}, an (optional) httpd service, and
* the (required) {@link ReportTask}.
@@ -1008,8 +1005,6 @@
* {@link LoadBalancer} service know which services exist, which is
* important for some of its functions.
* <p>
- * Once these task(s) have been started, this task will throw an exception
- * in order to prevent it from being re-executed.
*
* FIXME This should explicitly await jini registrar discovery, zookeeper
* client connected, and whatever other preconditions must be statisified
@@ -1032,8 +1027,7 @@
*/
final long begin = System.currentTimeMillis();
- public StartDeferredTasksTask() {
-
+ private StartDeferredTasksTask() {
}
/**
@@ -1043,8 +1037,6 @@
*/
public void run() {
- final boolean started;
-
try {
// /*
@@ -1061,9 +1053,7 @@
// return;
//
// }
-
- started = startDeferredTasks();
-
+ startDeferredTasks();
} catch (Throwable t) {
log.warn("Problem in report task?", t);
@@ -1072,60 +1062,56 @@
}
- if (started) {
-
- /*
- * Note: This exception is thrown once this task has executed
- * successfully.
- */
-
- throw new RuntimeException("Normal completion.");
-
- }
-
}
/**
* Starts performance counter collection once the service {@link UUID}
* is known.
*
- * @return <code>true</code> iff performance counter collection was
- * started.
- *
* @throws IOException
* if {@link IDataService#getServiceUUID()} throws this
* exception (it never should since it is a local method
* call).
*/
- protected boolean startDeferredTasks() throws IOException {
+ protected void startDeferredTasks() throws IOException {
// elapsed time since we started running this task.
final long elapsed = System.currentTimeMillis() - begin;
- if (getServiceUUID() == null) {
-
+ // Wait for the service ID to become available, trying every
+ // two seconds, while logging failures.
+ while (true) {
+ if (getServiceUUID() != null) {
+ break;
+ }
if (elapsed > 1000 * 10)
log.warn(ERR_NO_SERVICE_UUID + " : iface="
+ getServiceIface() + ", name=" + getServiceName()
+ ", elapsed=" + elapsed);
else if (log.isInfoEnabled())
- log.info(ERR_NO_SERVICE_UUID);
-
- return false;
-
+ log.info(ERR_NO_SERVICE_UUID);
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ }
}
- if (!isServiceReady()) {
-
+ // Wait for the service to become ready, trying every
+ // two seconds, while logging failures.
+ while (true) {
+ if (isServiceReady()) {
+ break;
+ }
if (elapsed > 1000 * 10)
log.warn(ERR_SERVICE_NOT_READY + " : iface="
+ getServiceIface() + ", name=" + getServiceName()
+ ", elapsed=" + elapsed);
else if (log.isInfoEnabled())
log.info(ERR_SERVICE_NOT_READY + " : " + elapsed);
-
- return false;
-
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ }
}
/*
@@ -1153,9 +1139,6 @@
// notify delegates that deferred startup has occurred.
AbstractFederation.this.didStart();
-
- return true;
-
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/sparse/GlobalRowStoreHelper.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/sparse/GlobalRowStoreHelper.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/sparse/GlobalRowStoreHelper.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -32,8 +32,12 @@
import org.apache.log4j.Logger;
+import com.bigdata.btree.DefaultTupleSerializer;
import com.bigdata.btree.IIndex;
import com.bigdata.btree.IndexMetadata;
+import com.bigdata.btree.keys.ASCIIKeyBuilderFactory;
+import com.bigdata.btree.keys.CollatorEnum;
+import com.bigdata.btree.keys.KeyBuilder;
import com.bigdata.journal.IIndexManager;
import com.bigdata.journal.ITx;
import com.bigdata.journal.TimestampUtility;
@@ -99,6 +103,25 @@
indexMetadata
.setSplitHandler(LogicalRowSplitHandler.INSTANCE);
+ if (CollatorEnum.JDK.toString().equals(
+ System.getProperty(KeyBuilder.Options.COLLATOR))) {
+ /*
+ * The JDK RulesBasedCollator embeds nul bytes in the
+ * Unicode sort keys. This makes them unsuitable for the
+ * SparseRowStore, which can not locate the start of the
+ * column name if there are embedded nuls in a Unicode
+ * primary key. As a work around, this forces an ASCII
+ * collation sequence if the JDK collator is the
+ * default. This is not ideal since non-ascii
+ * distinctions will be lost, but it is better than
+ * being unable to decode the column names.
+ */
+ log.warn("Forcing ASCII collator.");
+ indexMetadata
+ .setTupleSerializer(new DefaultTupleSerializer(
+ new ASCIIKeyBuilderFactory()));
+ }
+
// Register the index.
indexManager.registerIndex(indexMetadata);
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -49,17 +49,22 @@
* prefix.
* <p>
* The encoded schema name is followed by the {@link KeyType#getByteCode()} and
- * then by a <code>nul</code> byte. By searching for the <code>nul</code>
- * byte we can identify the end of the encoded schema name and also the data
- * type of the primary key. Most kinds of primary keys have a fixed length
- * encoding, e.g., {@link Long}, {@link Double}, etc. However, Unicode primary
- * keys have a variable length encoding which makes life more ... complex. Since
- * the keys need to reflect the total sort order we can not include the byte
- * count of the primary key in the key itself. The only reasonable approach is
- * to append a byte sequence to the key that never occurs within the generated
- * Unicode sort keys. We use a <code>nul</code> byte for this purpose since it
- * is not emitted by most Unicode collation implementations as it would cause
- * grief for C-language strings.
+ * then by a <code>nul</code> byte. By searching for the <code>nul</code> byte
+ * we can identify the end of the encoded schema name and also the data type of
+ * the primary key. Most kinds of primary keys have a fixed length encoding,
+ * e.g., {@link Long}, {@link Double}, etc.
+ * <p>
+ * Unicode primary keys have a variable length encoding which makes life more
+ * complex. For Unicode primary keys, we break with the collation order and use
+ * the UTF8 encoding of the key. This means that the primary key can be decoded
+ * and preserves hierarchical namespace clustering within the row store but does
+ * not impose a total sort order per Unicode sort key semantics. The only
+ * reasonable approach is to append a byte sequence to the key that never occurs
+ * within the generated Unicode sort keys. Again, we use a <code>nul</code> byte
+ * to mark the end of the Unicode primary key since it is not emitted by most
+ * Unicode collation implementations as it would cause grief for C-language
+ * strings. (However, see SparseRowStore.Options#PRIMARY_KEY_UNICODE_CLEAN} for
+ * information on backward compatibility.)
*
* @see Schema#fromKey(IKeyBuilder, Object)
* @see KeyType#getKeyType(byte)
@@ -166,15 +171,15 @@
* The decoded primary key.
*
* @throws UnsupportedOperationException
- * if the primary key can not be decoded (e.g., for
- * {@link KeyType#Unicode} keys).
+ * if the primary key can not be decoded.
*/
public Object getPrimaryKey() {
if(primaryKey == null) {
-
- throw new UnsupportedOperationException("Can not decode: keyType="+primaryKeyType);
-
+
+ throw new UnsupportedOperationException("Can not decode: keyType="
+ + primaryKeyType);
+
}
return primaryKey;
@@ -220,14 +225,13 @@
* Note: the KeyType byte occurs after the schema name bytes and before
* the [nul].
*/
+ int primaryKeyOffset = 0;
{
boolean found = false;
int schemaBytesLength = 0;
- int primaryKeyOffset = 0;
-
for (int i = 0; i < key.length; i++) {
if (key[i] == (byte) 0) {
@@ -256,7 +260,6 @@
this.primaryKeyTypeOffset = schemaBytesLength;
- this.primaryKeyOffset = primaryKeyOffset;
// note: ArrayIndexOutOfBounds with index==-1 means ICU library not on classpath!
this.primaryKeyType = KeyType.getKeyType(KeyBuilder.decodeByte(key[primaryKeyTypeOffset]));
@@ -273,42 +276,48 @@
primaryKeyLength = primaryKeyType.getEncodedLength();
+ this.primaryKeyOffset = primaryKeyOffset;
+
columnNameOffset = primaryKeyOffset + primaryKeyLength;
} else {
/*
- * Scan for the next [nul] byte.
+ * Scan for the next [nul] byte (ASCII).
*/
boolean found = false;
int primaryKeyLength = 0;
-
+
for (int i = primaryKeyOffset; i < key.length; i++) {
if (key[i] == (byte) 0) {
primaryKeyLength = i - primaryKeyOffset;
-
+
found = true;
-
+
break;
-
+
}
}
-
- if(!found) {
+ if (!found) {
+
throw new RuntimeException(
"Could not locate the end of the encoded schema name: keyType="
- + primaryKeyType+", key="+BytesUtil.toString(key));
+ + primaryKeyType + ", key="
+ + BytesUtil.toString(key));
}
this.primaryKeyLength = primaryKeyLength;
- // Note: also skips the [nul] byte terminating the primary key.
+ this.primaryKeyOffset = primaryKeyOffset;
+
+ // Note: also skips the [nul] byte terminating the primary
+ // key.
this.columnNameOffset = primaryKeyOffset + primaryKeyLength + 1;
}
@@ -327,10 +336,26 @@
primaryKey = KeyBuilder.decodeFloat(key, primaryKeyOffset);
break;
case Unicode:
- /*
- * Note: Decode is not possible for this case.
- */
- primaryKey = null;
+ if (SparseRowStore.primaryKeyUnicodeClean) {
+ final byte[] bytes = new byte[primaryKeyLength];
+ System.arraycopy(key, primaryKeyOffset, bytes, 0, primaryKeyLength);
+ try {
+ primaryKey = new String(bytes, SparseRowStore.UTF8);
+ } catch (UnsupportedEncodingException ex) {
+ throw new RuntimeException(
+ "Could not decode the primary key"
+ + ": primaryKeyOffset="
+ + primaryKeyOffset
+ + ", primaryKeyLength="
+ + primaryKeyLength + ", key="
+ + BytesUtil.toString(key));
+ }
+ } else {
+ /*
+ * Note: Decode is not possible for this case.
+ */
+ primaryKey = null;
+ }
break;
case ASCII:
primaryKey = KeyBuilder.decodeASCII(key, primaryKeyOffset,
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java 2010-07-23 18:36:43 UTC (rev 3275)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java 2010-07-23 18:43:00 UTC (rev 3276)
@@ -154,7 +154,7 @@
/*
* Key builder stuff.
*/
-
+
/**
* Helper method appends a typed value to the compound key (this is used to
* get the primary key into the compound key).
@@ -172,8 +172,9 @@
*
* @see KeyDecoder
*/
- final protected IKeyBuilder appendPrimaryKey(IKeyBuilder keyBuilder, Object v, boolean successor) {
-
+ final protected IKeyBuilder appendPrimaryKey(final IKeyBuilder keyBuilder,
+ final Object v, final boolean successor) {
+
final KeyType keyType = getPrimaryKeyType();
if (successor) {
@@ -187,9 +188,27 @@
case Float:
return keyBuilder.append(successor(keyBuilder,((Number) v).floatValue()));
case Double:
- return keyBuilder.append(successor(keyBuilder,((Number) v).doubleValue()));
- case Unicode:
- return keyBuilder.appendText(v.toString(), true/*unicode*/, true/*successor*/).appendNul();
+ return keyBuilder.append(successor(keyBuilder, ((Number) v)
+ .doubleValue()));
+ case Unicode: {
+ final String tmp = v.toString();
+ ...
[truncated message content] |
|
From: <btm...@us...> - 2010-07-28 20:48:02
|
Revision: 3337
http://bigdata.svn.sourceforge.net/bigdata/?rev=3337&view=rev
Author: btmurphy
Date: 2010-07-28 20:47:53 +0000 (Wed, 28 Jul 2010)
Log Message:
-----------
[branch dev-btm]: trac #126 - Uses of InetAddress.getLocalAddress should be changed to a mechanism that returns a non-loopback address
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/MoveTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractEmbeddedLoadBalancerService.java
branches/dev-btm/bigdata/src/java/com/bigdata/util/config/NicUtil.java
branches/dev-btm/bigdata/src/test/com/bigdata/counters/httpd/TestCounterSetHTTPDServer.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestResourceService.java
branches/dev-btm/bigdata/src/test/com/bigdata/test/ExperimentDriver.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/config/executor.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/JiniServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/MaxClientServicesPerHostConstraint.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/MaxDataServicesPerHostConstraint.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ZookeeperServerConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/JiniCoreServicesProcessHelper.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/ZookeeperProcessHelper.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/config/loadbalancer.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/shardlocator.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/process/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/process/config/process.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/config/quorum.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/LoadBalancerServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/LookupStarter.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/config/lookup.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/config/shard.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/config/transaction.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/ConfigDeployUtil.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/default-deploy.properties
branches/dev-btm/bigdata-jini/src/java/com/bigdata/zookeeper/ZooHelper.java
branches/dev-btm/bigdata-jini/src/resources/config/bigdataStandaloneTesting.config
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/config/TestZookeeperServerEntry.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/testReggie.config
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/testStartJini.config
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/AbstractServerTestCase.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/zookeeper/testzoo.config
branches/dev-btm/bigdata-lubm/src/java/edu/lehigh/swat/bench/ubt/Test.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/metrics/TestMetrics.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/AbstractServerTestCase.java
branches/dev-btm/build.properties
branches/dev-btm/build.xml
branches/dev-btm/src/resources/bin/config/browser.config
branches/dev-btm/src/resources/bin/pstart
branches/dev-btm/src/resources/config/bigdataCluster.config
branches/dev-btm/src/resources/config/jini/reggie.config
branches/dev-btm/src/resources/scripts/bigdataup
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -48,6 +48,8 @@
import com.bigdata.counters.win.StatisticsCollectorForWindows;
import com.bigdata.io.DirectBufferPool;
import com.bigdata.rawstore.Bytes;
+import com.bigdata.util.config.ConfigDeployUtil;
+import com.bigdata.util.config.NicUtil;
import com.bigdata.util.httpd.AbstractHTTPD;
/**
@@ -64,7 +66,6 @@
* and Un*x platforms so as to support the declared counters on all platforms.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
abstract public class AbstractStatisticsCollector implements IStatisticsCollector {
@@ -83,32 +84,10 @@
String s;
try {
-
-// hostname = InetAddress.getLocalHost().getHostName();
-
- s = InetAddress.getLocalHost().getCanonicalHostName();
-
- } catch (UnknownHostException e) {
-
- try {
-
- s = InetAddress.getLocalHost().getHostName();
-
- } catch(UnknownHostException e2) {
-
- final String msg = "Could not resolve hostname";
- try {
- log.error(msg);
- } catch(Throwable t) {
- System.err.println(msg);
- }
-
- s = "localhost";
-
- //throw new AssertionError(e);
-
- }
-
+ s = NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
+ } catch(Throwable t) {//for now, maintain same failure logic as used previously
+ t.printStackTrace();
+ s = NicUtil.getIpAddressByLocalHost();
}
fullyQualifiedHostName = s;
@@ -520,7 +499,6 @@
* Options for {@link AbstractStatisticsCollector}
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public interface Options {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/resources/MoveTask.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/resources/MoveTask.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/resources/MoveTask.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -63,6 +63,8 @@
import com.bigdata.service.IMetadataService;
import com.bigdata.service.MetadataService;
import com.bigdata.service.ResourceService;
+import com.bigdata.util.config.ConfigDeployUtil;
+import com.bigdata.util.config.NicUtil;
/**
* Task moves an index partition to another {@link IDataService}.
@@ -155,7 +157,6 @@
* need to do the atomic update phase.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public class MoveTask extends AbstractPrepareTask<MoveResult> {
@@ -421,7 +422,6 @@
* successful.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*
* @todo optimization to NOT send an empty index segment if there are no
* buffered writes on the live journal.
@@ -434,6 +434,7 @@
private final UUID targetDataServiceUUID;
private final int targetIndexPartitionId;
private final Event parentEvent;
+ private final InetAddress thisInetAddr;
/**
*
@@ -481,6 +482,11 @@
this.targetIndexPartitionId = targetIndexPartitionId;
this.parentEvent = parentEvent;
+ try {
+ this.thisInetAddr = InetAddress.getByName(NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.dataNetwork"), false));
+ } catch(Throwable t) {
+ throw new IllegalArgumentException(t.getMessage(), t);
+ }
}
/**
@@ -591,7 +597,7 @@
targetIndexPartitionId,//
historicalWritesBuildResult.segmentMetadata,//
bufferedWritesBuildResult.segmentMetadata,//
- InetAddress.getLocalHost(),//
+ thisInetAddr,
resourceManager
.getResourceServicePort()//
)).get();
@@ -856,7 +862,6 @@
* until the "receive" operation is complete.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
private static class IsIndexRegistered_UsingWriteService implements
IIndexProcedure {
@@ -896,7 +901,6 @@
* @see InnerReceiveIndexPartitionTask
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
protected static class ReceiveIndexPartitionTask extends DataServiceCallable<Void> {
@@ -1039,7 +1043,6 @@
* source data service.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
private static class InnerReceiveIndexPartitionTask extends AbstractTask<Void> {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractEmbeddedLoadBalancerService.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractEmbeddedLoadBalancerService.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractEmbeddedLoadBalancerService.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -5,6 +5,9 @@
import java.util.Properties;
import java.util.UUID;
+import com.bigdata.util.config.ConfigDeployUtil;
+import com.bigdata.util.config.NicUtil;
+
/**
* Embedded {@link LoadBalancerService}.
*
@@ -14,7 +17,7 @@
abstract public class AbstractEmbeddedLoadBalancerService extends LoadBalancerService {
// final private UUID serviceUUID;
- final private String hostname;
+ private String hostname = NicUtil.getIpAddressByLocalHost();//for now, maintain the same failure logic as in constructor
public AbstractEmbeddedLoadBalancerService(UUID serviceUUID,
Properties properties) {
@@ -28,17 +31,11 @@
setServiceUUID(serviceUUID);
- String hostname;
try {
-
- hostname = Inet4Address.getLocalHost().getCanonicalHostName();
-
- } catch (UnknownHostException e) {
-
- hostname = "localhost";
-
+ this.hostname = NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
+ } catch(Throwable t) {
+ t.printStackTrace();
}
- this.hostname = hostname;
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/util/config/NicUtil.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/util/config/NicUtil.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/util/config/NicUtil.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -507,54 +507,42 @@
/**
* Special-purpose convenience method that returns a
* <code>String</code> value representing the ip address of
- * the current node; where the value that is returned is
- * determined according to following criteria:
+ * the current node.
* <p>
- * <ul>
- * <li> If a non-<code>null</code> value is input for the
- * <code>systemPropertyName</code> parameter, then
- * this is viewed as a declaration by the caller that
- * that the system property with that given value
- * should take precedence over all other means of
- * determining the desired ip address. As such, this
- * method determines if a system property having the
- * given has indeed been set and, if it has, returns
- * the ip address of the nic having that name; or
- * <code>null</code> if there is no nic with the
- * desired name installed on the node.
- * <li> If a non-<code>null</code> value is input for the
- * <code>systemPropertyName</code> parameter, but
- * no system property with that name has been set,
- * and <code>true</code> has been passed in for
- * the <code>fallbackOk</code> parameter, then this
- * method will return the IPV4 based address of the
- * first reachable nic that is found on the node.
- * Upon failing to find such an address, if the
- * <code>loopbackOk</code> parameter is also
- * <code>true</code>, then this method will return
- * the <i>loop back</i> address of the node; otherwise
- * <code>null</code> is returned.
- * <li> If <code>null</code> is input for the
- * <code>systemPropertyName</code> parameter, but
- * a non-<code>null</code> value is input for the
- * <code>defaultNic</code> parameter, then this
- * method returns the ip address of the nic having
- * that name; or <code>null</code> if there is no
- * nic with the desired default name installed on the
- * node.
- * <li> If <code>null</code> is input for both the
- * <code>systemPropertyName</code> parameter and the
- * <code>defaultNic</code> parameter, and if the
- * <code>fallbackOk</code> parameter is <code>true</code>,
- * then this method will return the IPV4 based address
- * of the first reachable nic that is found on the node.
- * Upon failing to find such an address, if the
- * <code>loopbackOk</code> parameter is also
- * <code>true</code>, then this method will return
- * the <i>loop back</i> address of the node; otherwise
- * <code>null</code> is returned.
- * </ul>
+ * If a non-<code>null</code> value is input for the
+ * <code>systemPropertyName</code> parameter, then this
+ * method first determines if a system property with
+ * name equivalent to the given value has been set and,
+ * if it has, returns the ip address of the nic whose name
+ * is equivalent to that system property value; or
+ * <code>null</code> if there is no nic with the desired
+ * name installed on the node.
* <p>
+ * If there is no system property whose name is the value
+ * of the <code>systemPropertyName</code> parameter, and
+ * if the value "default" is input for the
+ * <code>defaultNic</code> parameter, then this method
+ * will return the IPV4 based address of the first reachable
+ * nic that can be found on the node; otherwise, if a
+ * non-<code>null</code> value not equal to "default" is
+ * input for the the <code>defaultNic</code> parameter,
+ * then this method returns the ip address of the nic
+ * corresponding to that given name; or <code>null</code>
+ * if there is no such nic name installed on the node.
+ * <p>
+ * If, on the other hand, <code>null</code> is input for
+ * the <code>systemPropertyName</code> parameter, then
+ * this method will attempt to find the desired ip address
+ * using only the value of the <code>defaultNic</code>,
+ * and applying the same search criteria as described
+ * above.
+ * <p>
+ * Note that in all cases, if <code>true</code> is input
+ * for the <code>loopOk</code> parameter, then upon failing
+ * to find a valid ip address using the specified search
+ * mechanism, this method will return the <i>loop back</i>
+ * address; otherwise, <code>null</code> is returned.
+ * <p>
* This method can be called from within a configuration
* as well as from within program control.
*
@@ -568,33 +556,24 @@
* the name of the network interface
* whose ip address should be returned
* if <code>null</code> is input for the
+ * <code>systemPropertyName</code> parameter,
+ * or if there is no system property with
+ * name equivalent the value of the
* <code>systemPropertyName</code> parameter.
*
- * @param fallbackOk if <code>true</code>, then if either
- * no system property is set having the
- * name referenced by the
- * <code>systemPropertyName</code> parameter,
- * or if <code>null</code> is input for both
- * the <code>systemPropertyName</code>
- * parameter and the <code>defaultNic</code>
- * parameter, return the IPV4 based address
- * of the first reachable network interface
- * that can be found on the node.
+ * @param loopbackOk if <code>true</code>, then return the
+ * <i>loop back</i> address upon failure
+ * to find a valid ip address using the
+ * search criteria specified through the
+ * <code>systemPropertyName</code> and
+ * <code>defaultNic</code> parameters.
*
- * @param loopbackOk if <code>true</code>, and if <code>true</code>
- * is also input for the <code>fallbackOk</code>
- * parameter, then if this method attempts,
- * but fails, to find a valid IPV4 fallback
- * address, then the node's <i>loop back</i>
- * address is returned.
- *
* @return a <code>String</code> representing an ip address associated
* with the current node; where the value that is returned is
* determined according to the criteria described above.
*/
public static String getIpAddress(String systemPropertyName,
String defaultNic,
- boolean fallbackOk,
boolean loopbackOk)
throws SocketException, IOException
{
@@ -613,32 +592,30 @@
}
if(propSet) {
return getIpAddress(nicName, 0, loopbackOk);
- } else {//desired system property not set, try fallback
- if(fallbackOk) {
- return getDefaultIpv4Address(loopbackOk);
+ } else {//system property not set, try default and/or fallback
+ if(defaultNic != null) {
+ if( defaultNic.equals("default") ) {
+ return getDefaultIpv4Address(loopbackOk);
+ } else {
+ return getIpAddress(defaultNic, 0, loopbackOk);
+ }
} else {
return null;
}
}
- } else {//no system property name provided, try default nic
+ } else {//no system property name provided, try default
if(defaultNic != null) {
- return getIpAddress(defaultNic, 0, loopbackOk);
- } else {//no default nic provided, try fallback
- if(fallbackOk) {
+ if( defaultNic.equals("default") ) {
return getDefaultIpv4Address(loopbackOk);
} else {
- return null;
+ return getIpAddress(defaultNic, 0, loopbackOk);
}
+ } else {
+ return getIpAddress(null, loopbackOk);
}
}
}
- public static String getIpAddress()
- throws SocketException, IOException
- {
- return getIpAddress(null, null, true, true);
- }
-
/**
* Examines each address associated with each network interface
* card (nic) installed on the current node, and returns the
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/counters/httpd/TestCounterSetHTTPDServer.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/counters/httpd/TestCounterSetHTTPDServer.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/counters/httpd/TestCounterSetHTTPDServer.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -39,6 +39,7 @@
import com.bigdata.counters.Instrument;
import com.bigdata.counters.OneShotInstrument;
import com.bigdata.counters.PeriodEnum;
+import com.bigdata.util.config.NicUtil;
/**
* Utility class for testing {@link CounterSetHTTPD} or
@@ -62,12 +63,10 @@
CounterSet cset = root.makePath("localhost");
- cset.addCounter("hostname", new OneShotInstrument<String>(
- InetAddress.getLocalHost().getHostName()));
+ String localIpAddr = NicUtil.getIpAddress("default.nic", "default", true);
+ cset.addCounter("hostname", new OneShotInstrument<String>(localIpAddr));
+ cset.addCounter("ipaddr", new OneShotInstrument<String>(localIpAddr));
- cset.addCounter("ipaddr", new OneShotInstrument<String>(InetAddress
- .getLocalHost().getHostAddress()));
-
// 60 minutes of data : @todo replace with CounterSetBTree (no fixed limit).
final HistoryInstrument<Double> history1 = new HistoryInstrument<Double>(
new History<Double>(new Double[60], PeriodEnum.Minutes
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/service/TestResourceService.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/service/TestResourceService.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/service/TestResourceService.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -43,6 +43,7 @@
import com.bigdata.service.ResourceService.ReadResourceTask;
import com.bigdata.util.concurrent.DaemonThreadFactory;
+import com.bigdata.util.config.NicUtil;
/**
* Test verifies the ability to transmit a file using the
@@ -103,14 +104,15 @@
};
+ InetAddress thisInetAddr;
+ thisInetAddr = InetAddress.getByName(NicUtil.getIpAddress("default.nic", "default", true));
try {
service.awaitRunning(100, TimeUnit.MILLISECONDS);
assertTrue(service.isOpen());
- assertEquals(tmpFile, new ReadResourceTask(InetAddress
- .getLocalHost(), service.port, allowedUUID, tmpFile).call());
+ assertEquals(tmpFile, new ReadResourceTask(thisInetAddr, service.port, allowedUUID, tmpFile).call());
if (log.isInfoEnabled())
log.info(service.counters.getCounters());
@@ -182,6 +184,8 @@
final List<File> tempFiles = new LinkedList<File>();
+ InetAddress thisInetAddr;
+ thisInetAddr = InetAddress.getByName(NicUtil.getIpAddress("default.nic", "default", true));
try {
service.awaitRunning(100, TimeUnit.MILLISECONDS);
@@ -197,8 +201,7 @@
tempFiles.add(tmpFile);
- tasks.add(new ReadResourceTask(InetAddress.getLocalHost(),
- service.port, allowedUUID, tmpFile));
+ tasks.add(new ReadResourceTask(thisInetAddr, service.port, allowedUUID, tmpFile));
}
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/test/ExperimentDriver.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/test/ExperimentDriver.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/test/ExperimentDriver.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -68,6 +68,7 @@
import com.bigdata.journal.ProxyTestCase;
import com.bigdata.util.NV;
+import com.bigdata.util.config.NicUtil;
/**
* A harness for running comparison of different configurations.
@@ -1364,13 +1365,9 @@
props.setProperty("os.arch.cpus", ""+SystemUtil.numProcessors());
try {
-
- props.setProperty("host",InetAddress.getLocalHost().getHostName());
-
- } catch(UnknownHostException ex) {
- /*
- * ignore.
- */
+ props.setProperty( "host", NicUtil.getIpAddress("default.nic", "default", true) );
+ } catch(Throwable t) {
+ t.printStackTrace();
}
return props;
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/ServiceImpl.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/ServiceImpl.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/ServiceImpl.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -244,13 +244,9 @@
}
innerProxy = (PrivateInterface)serverExporter.export(this);
+ String hostname = NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
outerProxy = ServiceProxy.createProxy
- (innerProxy, proxyId,
- NicUtil.getIpAddress(
- System.getProperty(
- "exportNic",
- ConfigDeployUtil.getString(
- "node.serviceNetwork"))) );
+ (innerProxy, proxyId, hostname);
adminProxy = AdminProxy.createProxy(innerProxy, proxyId);
//Setup lookup discovery
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/config/executor.config
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/config/executor.config 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/config/executor.config 2010-07-28 20:47:53 UTC (rev 3337)
@@ -16,10 +16,8 @@
com.bigdata.executor {
- private static exportNic =
- System.getProperty("exportNic",
- ConfigDeployUtil.getString("node.serviceNetwork"));
- private static exportIpAddr = NicUtil.getIpAddress(exportNic);
+ private static exportIpAddr =
+ NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
private static exportPort =
Integer.parseInt( System.getProperty("exportPort", "0") );
@@ -59,7 +57,7 @@
net.jini.discovery.LookupDiscovery {
multicastRequestHost = com.bigdata.executor.exportIpAddr;
multicastInterfaces = new NetworkInterface[] {
- NicUtil.getNetworkInterface(com.bigdata.executor.exportNic)
+ NicUtil.getNetworkInterface(com.bigdata.executor.exportIpAddr)
};
}
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/JiniServiceConfiguration.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/JiniServiceConfiguration.java 2010-07-28 18:31:51 UTC (rev 3336)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/JiniServiceConfiguration.java 2010-07-28 20:47:53 UTC (rev 3337)
@@ -78,6 +78,8 @@
import com.bigdata.service.jini.JiniClient;
import com.bigdata.service.jini.JiniClientConfig;
import com.bigdata.service.jini.JiniFederation;
+import com.bigdata.util.config.ConfigDeployUtil;
+import com.bigdata.util.config.NicUtil;
import com.bigdata.zookeeper.ZNodeCreatedWatcher;
/**
@@ -130,6 +132,8 @@
public final Properties properties;
public final String[] jiniOptions;
+ private final String serviceIpAddr;
+
protected void toString(StringBuilder sb) {
super.toString(sb);
@@ -155,6 +159,7 @@
final Configuration config) throws ConfigurationException {
super(className, config);
+//BTM
System.out.println("*** JiniServiceConfiguration: constructor");
final JiniClientConfig tmp = new JiniClientConfig(className, config);
@@ -176,6 +181,12 @@
} else {
log.warn("groups = " + Arrays.toString(this.groups));
}
+
+ try {
+ this.serviceIpAddr = NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
+ } catch(IOException e) {
+ throw new ConfigurationException(e.getMessage(), e);
+ }
}
/**
@@ -472,8 +483,7 @@
final ServiceDir serviceDir = new ServiceDir(this.serviceDir);
- final Hostname hostName = new Hostname(InetAddress.getLocal...
[truncated message content] |
|
From: <btm...@us...> - 2010-07-29 12:58:08
|
Revision: 3340
http://bigdata.svn.sourceforge.net/bigdata/?rev=3340&view=rev
Author: btmurphy
Date: 2010-07-29 12:57:54 +0000 (Thu, 29 Jul 2010)
Log Message:
-----------
merge -r:3267:HEAD(3339) ~/trunk/bigdata ~/bigdata/branches/dev-btm [trunk --> branch dev-btm]
Modified Paths:
--------------
branches/dev-btm/.classpath
branches/dev-btm/bigdata/src/architecture/mergePriority.xls
branches/dev-btm/bigdata/src/java/com/bigdata/LRUNexus.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/BloomFilterFactory.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/Checkpoint.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/DefaultTupleSerializer.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexMetadata.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegment.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentBuilder.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentStore.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/NOPTupleSerializer.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/NodeSerializer.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/isolation/IsolatedFusedView.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/IKeyBuilder.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/proc/AbstractKeyArrayIndexProcedure.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/view/FusedView.java
branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java
branches/dev-btm/bigdata/src/java/com/bigdata/io/DirectBufferPool.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractBufferStrategy.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractJournal.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/BufferMode.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/CommitRecordIndex.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/FileMetadata.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/Name2Addr.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/Options.java
branches/dev-btm/bigdata/src/java/com/bigdata/mdi/IndexPartitionCause.java
branches/dev-btm/bigdata/src/java/com/bigdata/mdi/MetadataIndex.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/IRelation.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/accesspath/SameVariableConstraint.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/locator/DefaultResourceLocator.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/IPredicate.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/Rule.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/Var.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/DefaultRuleTaskFactory.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/RuleState.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/JoinTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/LocalJoinTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/JournalIndex.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/StoreManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/ViewMetadata.java
branches/dev-btm/bigdata/src/java/com/bigdata/search/ReadIndexTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/CommitTimeIndex.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ILoadBalancerService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/LoadBalancerService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/RawDataServiceTupleIterator.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/proxy/RemoteAsynchronousIteratorImpl.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/proxy/RemoteChunk.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/AbstractAtomicRowReadOrWrite.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/TPSTupleSerializer.java
branches/dev-btm/bigdata/src/java/com/bigdata/striterator/IKeyOrder.java
branches/dev-btm/bigdata/src/resources/logging/log4j.properties
branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractBTreeTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractIndexSegmentTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestAll_IndexSegment.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestBigdataMap.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexSegmentBuilderWithLargeTrees.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexSegmentBuilderWithSmallTree.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestLeafSplitShortestSeparatorKey.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/AbstractUnicodeKeyBuilderTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java
branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestAll.java
branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestRootBlockView.java
branches/dev-btm/bigdata/src/test/com/bigdata/relation/accesspath/TestSameVariableConstraint.java
branches/dev-btm/bigdata/src/test/com/bigdata/relation/rule/AbstractRuleTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/relation/rule/TestRule.java
branches/dev-btm/bigdata/src/test/com/bigdata/search/TestAll.java
branches/dev-btm/bigdata/src/test/com/bigdata/search/TestSearchRestartSafe.java
branches/dev-btm/bigdata/src/test/com/bigdata/sparse/TestAll.java
branches/dev-btm/bigdata/src/test/com/bigdata/util/concurrent/TestAll.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/AbstractServicesManagerService.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServicesManagerServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/AbstractServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/LoadBalancerServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/benchmark/ThroughputMaster.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/BroadcastSighup.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/axioms/Axioms.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/axioms/BaseAxioms.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/BackchainOwlSameAsIterator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/BackchainOwlSameAsPropertiesIterator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/BackchainOwlSameAsPropertiesPIterator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/BackchainOwlSameAsPropertiesPOIterator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/BackchainOwlSameAsPropertiesSPIterator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/BackchainOwlSameAsPropertiesSPOIterator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/BackchainTypeResourceIterator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/FullyBufferedJustificationIterator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/Justification.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/OwlSameAsPropertiesExpandingIterator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/RdfTypeRdfsResourceFilter.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/SPOAssertionBuffer.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/inf/TruthMaintenance.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/BigdataRDFFullTextIndex.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/ITermIndexCodes.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Id2TermTupleSerializer.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Id2TermWriteProc.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/KVOTermIdComparator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconKeyBuilder.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/ReverseIndexWriterTask.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdTupleSerializer.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdWriteProc.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdWriteTask.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/TermIdEncoder.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/VerifyStatementBuffer.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/magic/IMagicTuple.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/magic/IRISUtils.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/magic/MagicAccessPath.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/magic/MagicIndexWriteProc.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/magic/MagicKeyOrder.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/magic/MagicPredicate.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/magic/MagicRelation.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/magic/MagicTuple.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/magic/MagicTupleSerializer.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/BNodeContextFactory.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataBNodeImpl.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataLiteralImpl.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataResourceImpl.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataStatementImpl.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataURIImpl.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataValue.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataValueFactory.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataValueFactoryImpl.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataValueIdComparator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataValueImpl.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/StatementEnum.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/model/TermIdComparator2.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/NQuadsParser.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/StatementBuffer.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/AbstractRuleDistinctTermScan.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/AbstractRuleFastClosure_11_13.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/AbstractRuleFastClosure_3_5_6_7_9.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/AbstractRuleFastClosure_5_6_7_9.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/BackchainAccessPath.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/DoNotAddFilter.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/FastClosure.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/MatchRule.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/RDFJoinNexus.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/RuleFastClosure3.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/DefaultGraphSolutionExpander.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/DistinctMultiTermAdvancer.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/DistinctSPOIterator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/DistinctTermAdvancer.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/ExplicitSPOFilter.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/ISPO.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/InGraphBinarySearchFilter.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/InGraphHashSetFilter.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/InferredSPOFilter.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/JustificationRemover.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/JustificationTupleSerializer.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/NamedGraphSolutionExpander.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/NoAxiomFilter.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/OSPComparator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/POSComparator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPO.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOAccessPath.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOComparator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOFilter.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOIndexWriteProc.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOKeyOrder.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOPredicate.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPORelation.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOSortKeyBuilder.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOStarJoin.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOTupleSerializer.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/BigdataSolutionResolverator.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/BigdataStatementIteratorImpl.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/BigdataValueIteratorImpl.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/IRawTripleStore.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/ITripleStore.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/ScaleOutTripleStore.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/TripleStoreUtility.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/vocab/BaseVocabulary.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/vocab/Vocabulary.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/TestAll.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/axioms/TestAxioms.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestAddTerms.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestAll.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestComparators.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestCompletionScan.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestFullTextIndex.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestId2TermTupleSerializer.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestTerm2IdTupleSerializer.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestVocabulary.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/magic/TestIRIS.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/magic/TestMagicKeyOrderStrategy.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/magic/TestMagicStore.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/model/TestAll.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/TestRDFXMLInterchangeWithStatementIdentifiers.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestAll.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestBackchainOwlSameAsPropertiesIterator.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestBackchainTypeResourceIterator.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestDatabaseAtOnceClosure.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestDistinctTermScan.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestJustifications.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestOptionals.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestOwlSameAsPropertiesExpandingIterator.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestRuleExpansion.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestRuleFastClosure_11_13.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestRuleFastClosure_3_5_6_7_9.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestSlice.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestTruthMaintenance.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestDefaultGraphAccessPath.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPO.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOAccessPath.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOKeyCoders.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOKeyOrder.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOPredicate.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPORelation.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOStarJoin.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOTupleSerializer.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOValueCoders.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/AbstractDistributedTripleStoreTestCase.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/AbstractEmbeddedTripleStoreTestCase.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/AbstractTestCase.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/TestAll.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/TestBulkFilter.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/TestLocalTripleStoreTransactionSemantics.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/TestRestartSafe.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/TestStatementIdentifiers.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/TestTripleStore.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/TestTripleStoreBasics.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataConstructIterator.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStatistics.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl2.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailHelper.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailRepository.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailRepositoryConnection.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/FreeTextSearchExpander.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/HitConvertor.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlClient.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestAll.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailEvaluationStrategyImpl.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestJoinScope.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestNamedGraphs.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestProvenanceQuery.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestQuery.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestSetBinding.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest2.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/stress/LoadClosureAndQueryTest.java
branches/dev-btm/bigdata-sails/src/test/org/openrdf/query/parser/sparql/SPARQLQueryTest.java
branches/dev-btm/build.properties
branches/dev-btm/src/resources/config/bigdataCluster.config
branches/dev-btm/src/resources/config/bigdataCluster16.config
branches/dev-btm/src/resources/config/standalone/bigdataStandalone.config
branches/dev-btm/src/resources/scripts/archiveRun.sh
branches/dev-btm/src/resources/scripts/bigdata
branches/dev-btm/src/resources/scripts/extractCounters.sh
Added Paths:
-----------
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentMultiBlockIterator.java
branches/dev-btm/bigdata/src/releases/RELEASE_0_83_0.txt
branches/dev-btm/bigdata/src/releases/RELEASE_0_83_1.txt
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexSegmentMultiBlockIterators.java
branches/dev-btm/bigdata-perf/bsbm/
branches/dev-btm/bigdata-perf/bsbm/RWStore.properties
branches/dev-btm/bigdata-perf/bsbm/WORMStore.properties
branches/dev-btm/bigdata-perf/bsbm/build.properties
branches/dev-btm/bigdata-perf/bsbm/build.xml
branches/dev-btm/bigdata-perf/bsbm/lib/
branches/dev-btm/bigdata-perf/bsbm/lib/jdom.jar
branches/dev-btm/bigdata-perf/bsbm/lib/log4j-1.2.12.jar
branches/dev-btm/bigdata-perf/bsbm/lib/ssj.jar
branches/dev-btm/bigdata-perf/bsbm/src/
branches/dev-btm/bigdata-perf/bsbm/src/java/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/DateGenerator.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/Generator.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/NormalDistGenerator.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/NormalDistRangeGenerator.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/ParetoDistGenerator.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/RandomBucket.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/TextGenerator.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/generator/ValueGenerator.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/BSBMResource.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/Offer.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/Person.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/Producer.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/Product.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/ProductFeature.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/ProductType.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/RatingSite.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/Review.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/model/Vendor.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/qualification/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/qualification/Qualification.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/qualification/QualificationDefaultValues.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/qualification/QueryResult.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/NTriples.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/ObjectBundle.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/SQLSerializer.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/Serializer.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/TriG.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/Turtle.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/VirtSerializer.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/serializer/XMLSerializer.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/AbstractParameterPool.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/ClientManager.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/ClientThread.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/CompiledQuery.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/CompiledQueryMix.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/LocalSPARQLParameterPool.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/NetQuery.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/PreCalcParameterPool.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/Query.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/QueryMix.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/SPARQLConnection.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/SQLConnection.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/SQLParameterPool.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/ServerConnection.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/TestDriver.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/testdriver/TestDriverDefaultValues.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/tools/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/tools/ResultTransform.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/BSBM.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/DC.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/FOAF.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/ISO3166.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/RDF.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/RDFS.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/REV.java
branches/dev-btm/bigdata-perf/bsbm/src/java/benchmark/vocabulary/XSD.java
branches/dev-btm/bigdata-perf/bsbm/src/resources/
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/givennames.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/ignoreQueries.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query1.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query10.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query10desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query10valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query11.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query11desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query11valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query12.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query12desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query12valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query1desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query1valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query2.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query2desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query2valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query3.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query3desc.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query3valid.txt
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query4-original.txt
br...
[truncated message content] |
|
From: <btm...@us...> - 2010-07-29 17:30:33
|
Revision: 3350
http://bigdata.svn.sourceforge.net/bigdata/?rev=3350&view=rev
Author: btmurphy
Date: 2010-07-29 17:30:26 +0000 (Thu, 29 Jul 2010)
Log Message:
-----------
[branch dev-btm]: modified browser and disco-tool to fallback to a reasonable default nic on failure
Modified Paths:
--------------
branches/dev-btm/bigdata-jini/src/java/com/bigdata/disco/config/disco.config
branches/dev-btm/src/resources/bin/config/browser.config
branches/dev-btm/src/resources/bin/config/serviceStarter.config
branches/dev-btm/src/resources/bin/disco-tool
branches/dev-btm/src/resources/bin/pstart
branches/dev-btm/src/resources/config/jini/reggie.config
branches/dev-btm/src/resources/config/jini/zookeeper.config
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/disco/config/disco.config
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/disco/config/disco.config 2010-07-29 16:54:30 UTC (rev 3349)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/disco/config/disco.config 2010-07-29 17:30:26 UTC (rev 3350)
@@ -18,8 +18,8 @@
com.bigdata.disco.DiscoveryTool {
- private static exportIpAddr =
- NicUtil.getIpAddress("${exportNic}", 0, "${exportHost}");
+ private static exportIpAddr =
+ NicUtil.getIpAddress("default.nic", "default", true);
lookupLocatorConstraints = null;
Modified: branches/dev-btm/src/resources/bin/config/browser.config
===================================================================
--- branches/dev-btm/src/resources/bin/config/browser.config 2010-07-29 16:54:30 UTC (rev 3349)
+++ branches/dev-btm/src/resources/bin/config/browser.config 2010-07-29 17:30:26 UTC (rev 3350)
@@ -9,12 +9,15 @@
import com.bigdata.util.config.NicUtil;
com.sun.jini.example.browser {
- private static exportIpAddr = NicUtil.getIpAddress("${default.nic}",0,"${exportHost}",true);
+
+ private static exportIpAddr =
+ NicUtil.getIpAddress("default.nic", "default", true);
+
private static serverILFactory =
new BasicILFactory(
new BasicMethodConstraints(
new InvocationConstraints(
- new ConnectionRelativeTime(10000), null)),
+ new ConnectionRelativeTime(10000L), null)),
null);
listenerExporter = new BasicJeriExporter(TcpServerEndpoint.getInstance(exportIpAddr,0),
Modified: branches/dev-btm/src/resources/bin/config/serviceStarter.config
===================================================================
--- branches/dev-btm/src/resources/bin/config/serviceStarter.config 2010-07-29 16:54:30 UTC (rev 3349)
+++ branches/dev-btm/src/resources/bin/config/serviceStarter.config 2010-07-29 17:30:26 UTC (rev 3350)
@@ -4,7 +4,9 @@
* needs to start any of the Bigdata services, as
* well as the Jini lookup service and browser.
*/
+import com.bigdata.util.config.ConfigDeployUtil;
import com.bigdata.util.config.ConfigurationUtil;
+import com.bigdata.util.config.NicUtil;
import com.sun.jini.config.ConfigUtil;
import com.sun.jini.start.NonActivatableServiceDescriptor;
@@ -13,16 +15,21 @@
com.sun.jini.start {
+ private static codebaseHost =
+ NicUtil.getIpAddress("bigdata.codebase.host", ConfigDeployUtil.getString("node.serviceNetwork"), false);
+ private static codebasePort =
+ Integer.parseInt( System.getProperty("bigdata.codebase.port", "0") );
+ private static codebaseRootDir =
+ System.getProperty("bigdata.codebase.rootdDir", "." );
+
private static jskCodebase =
ConfigurationUtil.computeCodebase
- ( "${bigdata.codebase.host}",
- "jsk-dl.jar", Integer.parseInt("${bigdata.codebase.port}"),
- "${bigdata.codebase.rootDir}", "none" );
+ ( codebaseHost, "jsk-dl.jar",
+ codebasePort, codebaseRootDir, "none" );
private static servicePolicyFile =
"${appHome}${/}var${/}config${/}policy${/}service.policy";
-
// For starting HTTP codebase class server
private static httpdCodebase = "";
private static httpdPolicyFile = servicePolicyFile;
@@ -92,8 +99,7 @@
// For starting a shard (data) service
- private static shardCodebase = jskCodebase;
-
+ private static shardCodebase = jskCodebase;
private static shardClasspath = "${appHome}${/}lib${/}bigdata.jar";
private static shardImplName = "com.bigdata.shard.ServiceImpl";
private static shardConfig = "${appHome}${/}var${/}config${/}jini${/}shard.config";
@@ -198,9 +204,8 @@
// For starting a lookup service
private static reggieServerCodebase =
ConfigurationUtil.computeCodebase
- ( "${bigdata.codebase.host}",
- "reggie-dl.jar", Integer.parseInt("${bigdata.codebase.port}"),
- "${bigdata.codebase.rootDir}", "none" );
+ ( codebaseHost, "reggie-dl.jar",
+ codebasePort, codebaseRootDir, "none" );
private static reggieCodebase =
ConfigUtil.concat( new String[] { reggieServerCodebase, " ", jskCodebase } );
@@ -226,9 +231,8 @@
// For starting a Jini browser
private static browserServerCodebase =
ConfigurationUtil.computeCodebase
- ( "${bigdata.codebase.host}",
- "browser-dl.jar", Integer.parseInt("${bigdata.codebase.port}"),
- "${bigdata.codebase.rootDir}", "none" );
+ ( codebaseHost, "browser-dl.jar",
+ codebasePort, codebaseRootDir, "none" );
private static browserCodebase =
ConfigUtil.concat( new String[] { browserServerCodebase, " ", jskCodebase } );
Modified: branches/dev-btm/src/resources/bin/disco-tool
===================================================================
--- branches/dev-btm/src/resources/bin/disco-tool 2010-07-29 16:54:30 UTC (rev 3349)
+++ branches/dev-btm/src/resources/bin/disco-tool 2010-07-29 17:30:26 UTC (rev 3350)
@@ -71,16 +71,22 @@
Prints this help message.
Useful properties:
- exportNic=<interfacename>
+ default.nic=<interfacename>
Specifies the name of the network interface on which the
ServiceDiscoveryManager's remote event listener will be exported. This
- property takes precedence over entityExportHost. The default value is
- "eth0".
+ property takes precedence over exportHost. The default behavior
+ regarding this property (in conjunction with the use of NicUtil
+ in the disco.config configuration file to retrieve the interface's
+ associated ip address) is to direct the configuration file to use
+ the IPv4 address of the first active network inteface that can
+ be found on the system.
exportHost=<ipaddress|hostname>
- Specifies the IP address on which the ServiceDiscoveryManager's remote
- event listener will be exported. This property is not used unless
- exportNic is "".
+ Specifies the IP address on which the ServiceDiscoveryManager's
+ remote event listener will be exported. This property will be
+ employed only when the value of the default.nic property
+ is set to the empty string ("") or a value that does not
+ correspond to any of the network interfaces on the system.
Examples:
Show information about all services, discovered through all
@@ -158,7 +164,7 @@
java_props = {
"java.security.manager": "",
"java.net.preferIPv4Stack": "true",
- "exportNic": "eth0",
+ "default.nic": "${default.nic}",
"networkInterface": "all",
"exportHost": socket.gethostname() }
Modified: branches/dev-btm/src/resources/bin/pstart
===================================================================
--- branches/dev-btm/src/resources/bin/pstart 2010-07-29 16:54:30 UTC (rev 3349)
+++ branches/dev-btm/src/resources/bin/pstart 2010-07-29 17:30:26 UTC (rev 3350)
@@ -49,8 +49,12 @@
default.nic=<interfacename>
Specifies the name of the network interface to use by default
for service export and remote communication. This property
- takes precedence over exportHost. The default value for this
- property is "eth0".
+ takes precedence over exportHost. The default behavior
+ regarding this property (in conjunction with configuration
+ files that use NicUtil to retrieve the interface's associated
+ ip address) is to direct the configuration file to use the
+ IPv4 address of the first active network inteface that can
+ be found on the system.
exportHost=<ipaddress|hostname>
Specifies the IP address or host name to use when exporting
@@ -63,11 +67,15 @@
Specifies the network address of the codebase HTTP server. If the
value is an interface name, the IP address assigned to that interface
will be used. If the value is an IP address or hostname, that value
- will be used directly. The default value is "eth0".
+ will be used directly. The default behavior regarding this property
+ (in conjunction with configuration files that use NicUtil to
+ retrieve the interface's associated ip address) is to direct the
+ configuration file to use the IPv4 address of the first active
+ network inteface that can be found on the system.
bigdata.codebase.port=<n>
The port number on <bigdata.codebase.host> on which the HTTP class
- server is running.
+ server is listening.
"""
serviceNames = bigdataServiceMap.keys()
@@ -131,10 +139,10 @@
self.properties['java.security.debug'] = "off"
def setEntityProperties(self):
- self.properties['default.nic'] = "eth0"
+ self.properties['default.nic'] = "${default.nic}"
self.properties['exportHost'] = socket.gethostname()
self.properties['exportPort'] = "0"
- self.properties['bigdata.codebase.host'] = "eth0"
+ self.properties['bigdata.codebase.host'] = "${bigdata.codebase.host}"
self.properties['bigdata.codebase.port'] = "8081"
self.properties['log4j.configuration'] = os.path.join( \
@@ -461,8 +469,6 @@
if params.appHome == "NOT_SET":
params.appHome = dirname(dirname( os.path.abspath(sys.argv[0])) )
- # BTM - params.appHome = dirname(dirname( os.path.abspath(sys.argv[0])) )
-
# print "appHome=" + params.appHome
# Instiantate the object for the service
Modified: branches/dev-btm/src/resources/config/jini/reggie.config
===================================================================
--- branches/dev-btm/src/resources/config/jini/reggie.config 2010-07-29 16:54:30 UTC (rev 3349)
+++ branches/dev-btm/src/resources/config/jini/reggie.config 2010-07-29 17:30:26 UTC (rev 3350)
@@ -17,7 +17,6 @@
private static exportIpAddr =
NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
-
private static exportPort =
Integer.parseInt( System.getProperty("exportPort", "0") );
Modified: branches/dev-btm/src/resources/config/jini/zookeeper.config
===================================================================
--- branches/dev-btm/src/resources/config/jini/zookeeper.config 2010-07-29 16:54:30 UTC (rev 3349)
+++ branches/dev-btm/src/resources/config/jini/zookeeper.config 2010-07-29 17:30:26 UTC (rev 3350)
@@ -7,7 +7,6 @@
* not yet been released as part of the Hadoop Zookeeper
* codebase.
*/
-
import java.net.NetworkInterface;
import com.sun.jini.config.ConfigUtil;
@@ -20,20 +19,18 @@
import net.jini.core.discovery.LookupLocator;
import net.jini.discovery.LookupDiscoveryManager;
-import com.bigdata.util.config.NicUtil;
import com.bigdata.util.config.ConfigDeployUtil;
+import com.bigdata.util.config.NicUtil;
org.apache.zookeeper.server.quorum {
- private static exportNic =
- System.getProperty("exportNic",
- ConfigDeployUtil.getString("node.serviceNetwork"));
- private static exportIpAddr = NicUtil.getIpAddress(exportNic);
+ private static exportIpAddr =
+ NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
private static exportPort =
Integer.parseInt( System.getProperty("exportPort", "0") );
- groupsToJoin = ConfigDeployUtil.getGroupsToDiscover();
- locatorsToJoin = ConfigDeployUtil.getLocatorsToDiscover();
+ private static groupsToJoin = ConfigDeployUtil.getGroupsToDiscover();
+ private static locatorsToJoin = ConfigDeployUtil.getLocatorsToDiscover();
private static exporterTcpServerEndpoint =
TcpServerEndpoint.getInstance(exportIpAddr, exportPort);
@@ -41,9 +38,7 @@
new BasicILFactory(
new BasicMethodConstraints(
new InvocationConstraints(
- new ConnectionRelativeTime(
- ConfigDeployUtil.getLong(
- "rmi.connectTimeout")),
+ new ConnectionRelativeTime( ConfigDeployUtil.getLong("rmi.connectTimeout")),
null)),
null);
@@ -82,7 +77,7 @@
net.jini.discovery.LookupDiscovery {
multicastRequestHost = org.apache.zookeeper.server.quorum.exportIpAddr;
multicastInterfaces = new NetworkInterface[] {
- NicUtil.getNetworkInterface(org.apache.zookeeper.server.quorum.exportNic)
+ NicUtil.getNetworkInterface(org.apache.zookeeper.server.quorum.exportIpAddr)
};
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <btm...@us...> - 2010-07-29 20:31:20
|
Revision: 3363
http://bigdata.svn.sourceforge.net/bigdata/?rev=3363&view=rev
Author: btmurphy
Date: 2010-07-29 20:31:11 +0000 (Thu, 29 Jul 2010)
Log Message:
-----------
merge -r:3339:HEAD(3358) ~/trunk/bigdata ~/bigdata/branches/dev-btm [trunk --> branch dev-btm]
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/btree/BTree.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegment.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentBuilder.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentMultiBlockIterator.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/Options.java
branches/dev-btm/bigdata/src/java/com/bigdata/mdi/LocalPartitionMetadata.java
branches/dev-btm/bigdata/src/java/com/bigdata/mdi/PartitionLocator.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/CompactingMergeTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/IncrementalBuildTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/JoinIndexPartitionTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/MoveTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/OverflowManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/SplitIndexPartitionTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/SplitUtility.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/StoreManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/MetadataService.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexPartitionFencePosts.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexSegmentMultiBlockIterators.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestBuildTask.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestBuildTask2.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestMergeTask.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestOverflow.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestResourceManagerBootstrap.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestSegSplitter.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/DumpFederation.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/TermIdEncoder.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl2.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuads.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuadsAndPipelineJoins.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuadsAndPipelineJoinsWithoutInlining.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithSids.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithSidsWithoutInlining.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithoutSids.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestDescribe.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestOptionals.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestUnions.java
branches/dev-btm/build.properties
Added Paths:
-----------
branches/dev-btm/bigdata/src/releases/RELEASE_0_83_2.txt
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestOrderBy.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestSids.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/sids.rdf
branches/dev-btm/lgpl-utils/LEGAL/lgpl-utils-license.txt
Removed Paths:
-------------
branches/dev-btm/dsi-utils/LEGAL/LICENSE.txt
branches/dev-btm/lgpl-utils/LEGAL/LICENSE.txt
Property Changed:
----------------
branches/dev-btm/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/attr/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/disco/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/
branches/dev-btm/bigdata-perf/
branches/dev-btm/bigdata-perf/lubm/lib/
branches/dev-btm/bigdata-perf/lubm/src/resources/
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util/
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/
branches/dev-btm/dsi-utils/LEGAL/
branches/dev-btm/dsi-utils/lib/
branches/dev-btm/dsi-utils/src/
branches/dev-btm/dsi-utils/src/test/
branches/dev-btm/dsi-utils/src/test/it/
branches/dev-btm/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/osgi/
Property changes on: branches/dev-btm
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/LEXICON_REFACTOR_BRANCH:2633-3304
/branches/bugfix-btm:2594-3237
/branches/fko:3150-3194
/trunk:2575-2594,2596-2877,2882-2903,2910-3339
+ /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/LEXICON_REFACTOR_BRANCH:2633-3304
/branches/bugfix-btm:2594-3237
/branches/fko:3150-3194
/trunk:2575-2594,2596-2877,2882-2903,2910-3358
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/BTree.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/BTree.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/BTree.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -1339,8 +1339,8 @@
oldPmd.getLeftSeparatorKey(), //
oldPmd.getRightSeparatorKey(),//
newResources,//
- oldPmd.getIndexPartitionCause(),//
- "" // history is deprecated.
+ oldPmd.getIndexPartitionCause()//
+// "" // history is deprecated.
);
// update the local partition metadata on our cloned IndexMetadata.
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegment.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegment.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegment.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -623,7 +623,7 @@
*
* @throws IllegalArgumentException
* if the <i>key</i> is <code>null</code>.
- * @throws RUntimeException
+ * @throws RuntimeException
* if the key does not lie within the optional key-range
* constraints for an index partition.
*/
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentBuilder.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentBuilder.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentBuilder.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -1227,9 +1227,9 @@
pmd.getLeftSeparatorKey(),//
pmd.getRightSeparatorKey(),//
null, // No resource metadata for indexSegment.
- pmd.getIndexPartitionCause(),
- pmd.getHistory()+
- "build("+pmd.getPartitionId()+",compactingMerge="+compactingMerge+") "
+ pmd.getIndexPartitionCause()
+// ,pmd.getHistory()+
+// "build("+pmd.getPartitionId()+",compactingMerge="+compactingMerge+") "
)
);
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentMultiBlockIterator.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentMultiBlockIterator.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentMultiBlockIterator.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -151,6 +151,16 @@
*/
private int blockLength = 0;
+ /*
+ * Counters
+ */
+
+ /** The #of leaves read so far. */
+ private long leafReadCount = 0;
+
+ /** The #of blocks read so far. */
+ private long blockReadCount = 0;
+
/**
*
* @param seg
@@ -209,11 +219,25 @@
: seg.findLeafAddr(toKey));
if (pool.getBufferCapacity() < store.getCheckpoint().maxNodeOrLeafLength) {
+
/*
- * Leaves are invariably larger than nodes. If the buffers in the
- * pool are too small to hold the largest record in the index
- * segment then you can not use this iterator.
+ * If the buffers in the pool are too small to hold the largest
+ * record in the index segment then you can not use this iterator.
+ *
+ * Note: We presume that the largest record is therefore a leaf. In
+ * practice this will nearly always be true as nodes have relatively
+ * little metadata per tuple while leaves store the value associated
+ * with the tuple.
+ *
+ * Note: AbstractBTree checks for this condition before choosing
+ * this iterator.
*/
+
+ throw new UnsupportedOperationException(
+ "Record is larger than buffer: maxNodeOrLeafLength="
+ + store.getCheckpoint().maxNodeOrLeafLength
+ + ", bufferCapacity=" + pool.getBufferCapacity());
+
}
if (firstLeafAddr == 0L) {
@@ -345,7 +369,7 @@
throw new IllegalStateException();
if (currentLeaf == null) {
if (log.isTraceEnabled())
- log.trace("Reading first leaf");
+ log.trace("Reading initial leaf");
// acquire the buffer from the pool.
acquireBuffer();
// Read the first block.
@@ -355,6 +379,12 @@
// Return the first leaf.
return leaf;
}
+ if (currentLeaf.identity == lastLeafAddr) {
+ // No more leaves.
+ if (log.isTraceEnabled())
+ log.trace("No more leaves (end of key range)");
+ return null;
+ }
/*
* We need to return the next leaf. We get the address of the next leaf
* from the nextAddr field of the current leaf.
@@ -363,7 +393,7 @@
if (nextLeafAddr == 0L) {
// No more leaves.
if (log.isTraceEnabled())
- log.trace("No more leaves");
+ log.trace("No more leaves (end of segment)");
return null;
}
/*
@@ -411,20 +441,25 @@
throw new IllegalArgumentException();
// offset into the buffer.
- final int toff = (int)(offset - blockOffset);
+ final int offsetWithinBuffer = (int)(offset - blockOffset);
- if (log.isTraceEnabled())
- log.trace("addr=" + addr + "(" + store.toString(addr)
- + "), blockOffset=" + blockOffset+" toff="+toff);
-
// read only view of the leaf in the buffer.
final ByteBuffer tmp = buffer.asReadOnlyBuffer();
- tmp.limit(toff + nbytes);
- tmp.position(toff);
+ tmp.limit(offsetWithinBuffer + nbytes);
+ tmp.position(offsetWithinBuffer);
// decode byte[] as ILeafData.
final ILeafData data = (ILeafData) seg.nodeSer.decode(tmp);
-
+
+ leafReadCount++;
+
+ if (log.isTraceEnabled())
+ log
+ .trace("read leaf: leafReadCount=" + leafReadCount
+ + ", addr=" + addr + "(" + store.toString(addr)
+ + "), blockOffset=" + blockOffset
+ + " offsetWithinBuffer=" + offsetWithinBuffer);
+
// return as Leaf.
return new ImmutableLeaf(seg, addr, data);
@@ -470,6 +505,14 @@
// the #of bytes that we will actually read.
final int nbytes = (int) Math.min(lastOffset - startOffset, b
.capacity());
+ if(log.isTraceEnabled())
+ log.trace("leafAddr=" + store.toString(leafAddr) + ", startOffset="
+ + startOffset + ", lastOffset=" + lastOffset + ", nbytes="
+ + nbytes);
+ if (nbytes == 0) {
+ throw new AssertionError("nbytes=0 : leafAddr"
+ + store.toString(leafAddr) + " : " + this);
+ }
// set the position to zero.
b.position(0);
// set the limit to the #of bytes to be read.
@@ -483,9 +526,29 @@
// update the offset/length in the store for the in memory block
blockOffset = startOffset;
blockLength = nbytes;
+ blockReadCount++;
if (log.isTraceEnabled())
- log.trace("leafAddr=" + leafAddr + ", blockOffset=" + blockOffset
- + ", blockLength=" + blockLength);
+ log.trace("read block: blockReadCount=" + blockReadCount
+ + ", leafAddr=" + store.toString(leafAddr)
+ + ", blockOffset=" + blockOffset + ", blockLength="
+ + blockLength);
}
+ public String toString() {
+ return super.toString() + //
+ "{file=" + store.getFile() + //
+ ",checkpoint="+store.getCheckpoint()+//
+ ",fromKey="+BytesUtil.toString(fromKey)+//
+ ",toKey="+BytesUtil.toString(toKey)+//
+ ",firstLeafAddr=" + store.toString(firstLeafAddr) + //
+ ",lastLeafAddr=" + store.toString(lastLeafAddr) + //
+ ",currentLeaf=" + (currentLeaf!=null?store.toString(currentLeaf.identity):"N/A") + //
+ ",blockOffset="+blockOffset+//
+ ",blockLength="+blockLength+//
+ ",bufferCapacity="+pool.getBufferCapacity()+//
+ ",leafReadCount="+leafReadCount+//
+ ",blockReadCount="+blockReadCount+//
+ "}";
+ }
+
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -470,6 +470,24 @@
}
}
+
+ /**
+ * Need to override commit to ensure the writeCache is flushed prior to
+ * writing the root block.
+ *
+ * For the DiskOnlyStrategy flushing the writeCache also ensures the backing
+ * file is created if the file is temporary.
+ *
+ * Note that the internal call to flush the writeCache must be synchronized
+ * or concurrent writers to the cache will cause problems.
+ */
+ public void commit() {
+ if (writeCache != null) {
+ synchronized(this) {
+ flushWriteCache();
+ }
+ }
+ }
/**
* Writes the {@link #writeCache} through to the disk and its position is
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/Options.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/Options.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/Options.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -402,28 +402,23 @@
String FORCE_ON_COMMIT = AbstractJournal.class.getName()+".forceOnCommit";
/**
- * This boolean option causes application data to be forced to stable
- * storage <em>before</em> we update the root blocks. This option seeks to
- * guarantee that the application data is stable on the disk before the
- * atomic commit. Some operating systems and/or file systems may otherwise
- * choose an ordered write or otherwise process the writes in a different
- * order. This could have the consequence that the root blocks are laid down
- * on the disk before the application data. In this situation a hard failure
- * during the write could result in the loss of application data since the
- * updated root blocks represent the atomic commit point but not all
- * application data was successfully made stable on disk.
+ * This boolean option may be used to request that application data are
+ * forced to stable storage <em>before</em> we update the root blocks
+ * (default {@value #DEFAULT_DOUBLE_SYNC}). This is accomplished by invoking
+ * {@link FileChannel#force(boolean)} before root blocks are updated as part
+ * of the atomic commit protocol in an attempt to guard against operating
+ * systems and/or file systems which may otherwise reorders writes with the
+ * consequence that the root blocks are laid down on the disk before the
+ * application data. In this situation a hard failure during the root block
+ * write could result in the loss of application data since the updated root
+ * blocks represent the atomic commit point but not all application data was
+ * successfully made stable on disk. However, note that there are often
+ * multiple cache layers in use by the operating system, the disk
+ * controller, and the disk. Therefore durability is thus best achieved
+ * through a mixture of methods, which can include battery powered hardware
+ * write cache and/or replication.
*
- * @deprecated This option does NOT provide a sufficient guarantee when a
- * write cache is in use by the operating system or the disk if
- * the layered write caches return before all data is safely on
- * disk (or in a battery powered cache). In order to protect
- * against this you MUST disable the write cache layers in the
- * operating system and the disk drive such that
- * {@link FileChannel#force(boolean)} will not return until the
- * data are in fact on stable storage. If you disable the OS and
- * disk write cache then you do NOT need to specify this option
- * since writes will be ordered and all data will be on disk
- * before we update the commit blocks.
+ * @see #DEFAULT_DOUBLE_SYNC
*/
String DOUBLE_SYNC = AbstractJournal.class.getName()+".doubleSync";
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/mdi/LocalPartitionMetadata.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/mdi/LocalPartitionMetadata.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/mdi/LocalPartitionMetadata.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -41,7 +41,6 @@
import com.bigdata.btree.IndexSegmentStore;
import com.bigdata.journal.Journal;
import com.bigdata.service.DataService;
-import com.bigdata.service.Event;
/**
* An immutable object providing metadata about a local index partition,
@@ -80,6 +79,9 @@
/**
*
* @see #getSourcePartitionId()
+ *
+ * @deprecated MoveTask manages without this field (it was required by the
+ * previous MOVE implementation).
*/
private int sourcePartitionId;
@@ -111,53 +113,53 @@
*/
private IndexPartitionCause cause;
- /**
- * A history of operations giving rise to the current partition metadata.
- * E.g., register(timestamp), copyOnOverflow(timestamp), split(timestamp),
- * join(partitionId,partitionId,timestamp), etc. This is truncated when
- * serialized to keep it from growing without bound.
- *
- * @deprecated See {@link #getHistory()}
- */
- private String history;
+// /**
+// * A history of operations giving rise to the current partition metadata.
+// * E.g., register(timestamp), copyOnOverflow(timestamp), split(timestamp),
+// * join(partitionId,partitionId,timestamp), etc. This is truncated when
+// * serialized to keep it from growing without bound.
+// *
+// * @deprecated See {@link #getHistory()}
+// */
+// private String history;
+//
+// /**
+// * If the history string exceeds {@link #MAX_HISTORY_LENGTH} characters then
+// * truncates it to the last {@link #MAX_HISTORY_LENGTH}-3 characters,
+// * prepends "...", and returns the result. Otherwise returns the entire
+// * history string.
+// *
+// * @deprecated See {@link #history}
+// */
+// protected String getTruncatedHistory() {
+//
+// if (MAX_HISTORY_LENGTH == 0)
+// return "";
+//
+// String history = this.history;
+//
+// if(history.length() > MAX_HISTORY_LENGTH) {
+//
+// /*
+// * Truncate the history.
+// */
+//
+// final int len = history.length();
+//
+// final int fromIndex = len - (MAX_HISTORY_LENGTH - 3);
+//
+// assert fromIndex > 0 : "len=" + len + ", fromIndex=" + fromIndex
+// + ", maxHistoryLength=" + MAX_HISTORY_LENGTH;
+//
+// history = "..." + history.substring(fromIndex, len);
+//
+// }
+//
+// return history;
+//
+// }
/**
- * If the history string exceeds {@link #MAX_HISTORY_LENGTH} characters then
- * truncates it to the last {@link #MAX_HISTORY_LENGTH}-3 characters,
- * prepends "...", and returns the result. Otherwise returns the entire
- * history string.
- *
- * @deprecated See {@link #history}
- */
- protected String getTruncatedHistory() {
-
- if (MAX_HISTORY_LENGTH == 0)
- return "";
-
- String history = this.history;
-
- if(history.length() > MAX_HISTORY_LENGTH) {
-
- /*
- * Truncate the history.
- */
-
- final int len = history.length();
-
- final int fromIndex = len - (MAX_HISTORY_LENGTH - 3);
-
- assert fromIndex > 0 : "len=" + len + ", fromIndex=" + fromIndex
- + ", maxHistoryLength=" + MAX_HISTORY_LENGTH;
-
- history = "..." + history.substring(fromIndex, len);
-
- }
-
- return history;
-
- }
-
- /**
* De-serialization constructor.
*/
public LocalPartitionMetadata() {
@@ -199,21 +201,21 @@
* the remote {@link DataService} will fill it in on arrival.
* @param cause
* The underlying cause for the creation of the index partition.
- * @param history
- * A human interpretable history of the index partition. The
- * history is a series of whitespace delimited records each of
- * more or less the form <code>foo(x,y,z)</code>. The history
- * gets truncated when the {@link LocalPartitionMetadata} is
- * serialized in order to prevent it from growing without bound.
*/
+// * @param history
+// * A human interpretable history of the index partition. The
+// * history is a series of whitespace delimited records each of
+// * more or less the form <code>foo(x,y,z)</code>. The history
+// * gets truncated when the {@link LocalPartitionMetadata} is
+// * serialized in order to prevent it from growing without bound.
public LocalPartitionMetadata(//
final int partitionId,//
final int sourcePartitionId,//
final byte[] leftSeparatorKey,//
final byte[] rightSeparatorKey,//
final IResourceMetadata[] resources,//
- final IndexPartitionCause cause,
- final String history
+ final IndexPartitionCause cause
+// final String history
) {
/*
@@ -232,7 +234,7 @@
this.cause = cause;
- this.history = history;
+// this.history = history;
/*
* Test arguments.
@@ -440,23 +442,23 @@
}
- /**
- * A history of the changes to the index partition.
- *
- * @deprecated I've essentially disabled the history (it is always empty
- * when it is persisted). I found it nearly impossible to read.
- * There are much saner ways to track what is going on in the
- * federation. An analysis of the {@link Event} log is much more
- * useful. If nothing else, you could examine the index
- * partition in the metadata index by scanning the commit points
- * and reading its state in each commit and reporting all state
- * changes.
- */
- final public String getHistory() {
-
- return history;
-
- }
+// /**
+// * A history of the changes to the index partition.
+// *
+// * @deprecated I've essentially disabled the history (it is always empty
+// * when it is persisted). I found it nearly impossible to read.
+// * There are much saner ways to track what is going on in the
+// * federation. An analysis of the {@link Event} log is much more
+// * useful. If nothing else, you could examine the index
+// * partition in the metadata index by scanning the commit points
+// * and reading its state in each commit and reporting all state
+// * changes.
+// */
+// final public String getHistory() {
+//
+// return history;
+//
+// }
final public int hashCode() {
@@ -466,7 +468,7 @@
}
// Note: used by assertEquals in the test cases.
- public boolean equals(Object o) {
+ public boolean equals(final Object o) {
if (this == o)
return true;
@@ -520,7 +522,7 @@
", rightSeparator="+BytesUtil.toString(rightSeparatorKey)+
", resourceMetadata="+Arrays.toString(resources)+
", cause="+cause+
- ", history="+history+
+// ", history="+history+
"}"
;
@@ -537,6 +539,17 @@
* but that field is only serialized for a journal.
*/
private static final transient short VERSION1 = 0x1;
+
+ /**
+ * This version serializes the {@link #partitionId} as 32-bits clean and
+ * gets rid of the <code>history</code> field.
+ */
+ private static final transient short VERSION2 = 0x2;
+
+ /**
+ * The current version.
+ */
+ private static final transient short VERSION = VERSION2;
public void readExternal(final ObjectInput in) throws IOException,
ClassNotFoundException {
@@ -546,13 +559,18 @@
switch (version) {
case VERSION0:
case VERSION1:
+ case VERSION2:
break;
default:
throw new IOException("Unknown version: " + version);
}
-
- partitionId = (int) LongPacker.unpackLong(in);
+ if (version < VERSION2) {
+ partitionId = (int) LongPacker.unpackLong(in);
+ } else {
+ partitionId = in.readInt();
+ }
+
sourcePartitionId = in.readInt(); // MAY be -1.
final int nresources = ShortPacker.unpackShort(in);
@@ -579,7 +597,9 @@
cause = (IndexPartitionCause)in.readObject();
- history = in.readUTF();
+ if (version < VERSION2) {
+ /* history = */in.readUTF();
+ }
resources = nresources>0 ? new IResourceMetadata[nresources] : null;
@@ -613,9 +633,13 @@
public void writeExternal(final ObjectOutput out) throws IOException {
- ShortPacker.packShort(out, VERSION1);
+ ShortPacker.packShort(out, VERSION);
- LongPacker.packLong(out, partitionId);
+ if (VERSION < VERSION2) {
+ LongPacker.packLong(out, partitionId);
+ } else {
+ out.writeInt(partitionId);
+ }
out.writeInt(sourcePartitionId); // MAY be -1.
@@ -640,7 +664,9 @@
out.writeObject(cause);
- out.writeUTF(getTruncatedHistory());
+ if (VERSION < VERSION2) {
+ out.writeUTF("");// getTruncatedHistory()
+ }
/*
* Note: we serialize using the IResourceMetadata interface so that we
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/mdi/PartitionLocator.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/mdi/PartitionLocator.java 2010-07-29 20:28:05 UTC (rev 3362)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/mdi/PartitionLocator.java 2010-07-29 20:31:11 UTC (rev 3363)
@@ -224,19 +224,36 @@
}
+ /**
+ * The original version.
+ */
private static final transient short VERSION0 = 0x0;
+
+ /**
+ * The {@link #partitionId} is now 32-bits clean.
+ */
+ private static final transient short VERSION1 = 0x0;
+ /**
+ * The current version.
+ */
+ private static final transient short VERSION = VERSION1;
+
public void readExternal(Ob...
[truncated message content] |
|
From: <btm...@us...> - 2010-07-30 13:51:55
|
Revision: 3372
http://bigdata.svn.sourceforge.net/bigdata/?rev=3372&view=rev
Author: btmurphy
Date: 2010-07-30 13:51:43 +0000 (Fri, 30 Jul 2010)
Log Message:
-----------
merge -r:3358:HEAD(3370) ~/bigdata/trunk ~/bigdata/branches/dev-btm [trunk --> branch dev-btm]
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BlobOverflowHandler.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
branches/dev-btm/bigdata/src/java/com/bigdata/cache/HardReferenceGlobalLRU.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java
branches/dev-btm/bigdata/src/test/com/bigdata/cache/TestAll.java
branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestConcurrentJournal.java
branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestJournalBasics.java
branches/dev-btm/bigdata/src/test/com/bigdata/relation/locator/TestDefaultResourceLocator.java
branches/dev-btm/bigdata/src/test/com/bigdata/search/TestKeyBuilder.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestBasicIndexStuff.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestDistributedTransactionService.java
branches/dev-btm/bigdata/src/test/com/bigdata/sparse/TestKeyEncodeDecode.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithSids.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithSidsWithoutInlining.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataConnectionTest.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataStoreTest.java
Removed Paths:
-------------
branches/dev-btm/CVSROOT/
branches/dev-btm/bigdata-master-pom/
Property Changed:
----------------
branches/dev-btm/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/attr/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/disco/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/
branches/dev-btm/bigdata-perf/
branches/dev-btm/bigdata-perf/lubm/lib/
branches/dev-btm/bigdata-perf/lubm/src/resources/
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util/
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/
branches/dev-btm/dsi-utils/LEGAL/
branches/dev-btm/dsi-utils/lib/
branches/dev-btm/dsi-utils/src/
branches/dev-btm/dsi-utils/src/test/
branches/dev-btm/dsi-utils/src/test/it/
branches/dev-btm/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/osgi/
Property changes on: branches/dev-btm
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/LEXICON_REFACTOR_BRANCH:2633-3304
/branches/bugfix-btm:2594-3237
/branches/fko:3150-3194
/trunk:2575-2594,2596-2877,2882-2903,2910-3358
+ /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/LEXICON_REFACTOR_BRANCH:2633-3304
/branches/bugfix-btm:2594-3237
/branches/fko:3150-3194
/trunk:2575-2594,2596-2877,2882-2903,2910-3370
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BlobOverflowHandler.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BlobOverflowHandler.java 2010-07-30 13:51:36 UTC (rev 3371)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BlobOverflowHandler.java 2010-07-30 13:51:43 UTC (rev 3372)
@@ -6,8 +6,10 @@
import com.bigdata.btree.IOverflowHandler;
import com.bigdata.btree.ITuple;
+import com.bigdata.btree.keys.IKeyBuilder;
import com.bigdata.btree.keys.KeyBuilder;
import com.bigdata.io.DataOutputBuffer;
+import com.bigdata.rawstore.Bytes;
import com.bigdata.rawstore.IBlock;
import com.bigdata.rawstore.IRawStore;
@@ -33,7 +35,7 @@
}
- DataOutputBuffer buf;
+ private transient DataOutputBuffer buf;
public void close() {
@@ -62,6 +64,8 @@
}
+ final IKeyBuilder keyBuilder = new KeyBuilder(Bytes.SIZEOF_LONG);
+
if (addr == 0L) {
/*
@@ -69,7 +73,7 @@
* their address.
*/
- return KeyBuilder.asSortKey(0L);
+ return keyBuilder.append(0L).getKey();
}
@@ -143,7 +147,7 @@
}
// the address of the block on the target store.
- return KeyBuilder.asSortKey(addr2);
+ return keyBuilder.append(addr2).getKey();
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-07-30 13:51:36 UTC (rev 3371)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-07-30 13:51:43 UTC (rev 3372)
@@ -1081,17 +1081,25 @@
* Note: This method is thread-safe.
* <p>
* Note: Strings are Unicode safe for the default locale. See
- * {@link Locale#getDefault()}. If you require a specific local or
- * different locals at different times or for different indices then you
- * MUST provision and apply your own {@link KeyBuilder}.
+ * {@link Locale#getDefault()}. If you require a specific local or different
+ * locals at different times or for different indices then you MUST
+ * provision and apply your own {@link KeyBuilder}.
*
* @param val
* An application key.
*
- * @return The unsigned byte[] equivilent of that key. This will be
- * <code>null</code> iff the <i>key</i> is <code>null</code>.
- * If the <i>key</i> is a byte[], then the byte[] itself will be
- * returned.
+ * @return The unsigned byte[] equivalent of that key. This will be
+ * <code>null</code> iff the <i>key</i> is <code>null</code>. If the
+ * <i>key</i> is a byte[], then the byte[] itself will be returned.
+ *
+ * @deprecated This method circumvents explicit configuration of the
+ * {@link KeyBuilder} and is used nearly exclusively by unit
+ * tests. While explicit configuration is not required for keys
+ * which do not include Unicode sort key components, this method
+ * also relies on a single global {@link KeyBuilder} instance
+ * protected by a lock. That lock is therefore a bottleneck. The
+ * correct practice is to use thread-local or per task
+ * {@link IKeyBuilder}s to avoid lock contention.
*/
@SuppressWarnings("unchecked")
public static final byte[] asSortKey(Object val) {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/cache/HardReferenceGlobalLRU.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/cache/HardReferenceGlobalLRU.java 2010-07-30 13:51:36 UTC (rev 3371)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/cache/HardReferenceGlobalLRU.java 2010-07-30 13:51:43 UTC (rev 3372)
@@ -57,7 +57,8 @@
* iterator. When compared to the {@link HardReferenceGlobalLRURecycler}, this
* implementation has approximately 10% higher throughput.
*
- * @version $Id$
+ * @version $Id: HardReferenceGlobalLRU.java 2799 2010-05-11 21:04:43Z
+ * thompsonbry $
* @author <a href="mailto:tho...@us...">Bryan Thompson
* </a>
* @param <K>
@@ -69,6 +70,8 @@
* {@link IDataRecordAccess} since we can not measure the bytesInMemory
* for those objects and hence the LRU eviction policy will not account
* for their memory footprint?
+ *
+ * @deprecated This implementation is not used.
*/
public class HardReferenceGlobalLRU<K, V> implements
IHardReferenceGlobalLRU<K, V> {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java 2010-07-30 13:51:36 UTC (rev 3371)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java 2010-07-30 13:51:43 UTC (rev 3372)
@@ -73,6 +73,9 @@
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
+ *
+ * @todo The key is now 100% decodable. The package should be updated to take
+ * advantage of that.
*/
public class KeyDecoder {
@@ -157,6 +160,22 @@
return a;
}
+
+ /**
+ * Return the schema name.
+ *
+ * @throws UnsupportedOperationException
+ * unless {@link SparseRowStore#schemaNameUnicodeClean} is
+ * <code>true</code>.
+ */
+ public String getSchemaName() {
+
+ if(!SparseRowStore.schemaNameUnicodeClean)
+ throw new UnsupportedOperationException();
+
+ return new String(getSchemaBytes());
+
+ }
/**
* The decoded {@link KeyType} for the primary key.
@@ -485,10 +504,17 @@
*/
public String toString() {
- return "KeyDecoder{primaryKeyType=" + primaryKeyType + ",col=" + col
- + ",timestamp=" + timestamp + ",key=" + BytesUtil.toString(key)
+ return "KeyDecoder{"
+ + (SparseRowStore.schemaNameUnicodeClean ? "schema="
+ + getSchemaName() + "," : "")//
+ + "primaryKeyType="+ primaryKeyType//
+ + (SparseRowStore.primaryKeyUnicodeClean ? ",primaryKey="
+ + getPrimaryKey() : "")//
+ + ",col=" + col //
+ + ",timestamp=" + timestamp //
+ + ",key=" + BytesUtil.toString(key) //
+ "}";
}
-
+
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java 2010-07-30 13:51:36 UTC (rev 3371)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java 2010-07-30 13:51:43 UTC (rev 3372)
@@ -128,14 +128,22 @@
if (schemaBytes == null) {
- /*
- * One time encoding of the schema name as a Unicode sort key.
- */
-
- schemaBytes = KeyBuilder.asSortKey(name);
+ if (SparseRowStore.schemaNameUnicodeClean) {
+ /*
+ * One time encoding of the schema name as UTF8.
+ */
+ try {
+ schemaBytes = name.getBytes(SparseRowStore.UTF8);
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ /*
+ * One time encoding of the schema name as a Unicode sort key.
+ */
+ schemaBytes = KeyBuilder.asSortKey(name);
+ }
-// schemaBytes = KeyBuilder.newInstance().append(name).append("\0").getKey();
-
}
return schemaBytes;
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java 2010-07-30 13:51:36 UTC (rev 3371)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java 2010-07-30 13:51:43 UTC (rev 3372)
@@ -1048,11 +1048,35 @@
*
* @author <a href="mailto:tho...@us...">Bryan
* Thompson</a>
- * @version $Id$
*/
public interface Options {
/**
+ * The schema name was originally written using a Unicode sort key.
+ * However, the JDK can generate Unicode sort keys with embedded nuls
+ * which in turn will break the logic to detect the end of the schema
+ * name in the key. In order to accommodate this behavior, the schema
+ * name is now encoded as UTF8 which also has the advantage that we can
+ * decode the schema name. Standard prefix compression on the B+Tree
+ * should make up for the larger representation of the schema name in
+ * the B+Tree.
+ * <p>
+ * This change was introduced on 7/29/2010 in the trunk. When this
+ * property is <code>true</code> it breaks compatibility with earlier
+ * revisions of the {@link SparseRowStore}. This flag may be set to
+ * <code>false</code> for backward compatibility.
+ *
+ * @see #DEFAULT_SCHEMA_NAME_UNICODE_CLEAN
+ */
+ String SCHEMA_NAME_UNICODE_CLEAN = Schema.class.getName()
+ + ".schemaName.unicodeClean";
+
+ /**
+ * @see https://sourceforge.net/apps/trac/bigdata/ticket/107
+ */
+ String DEFAULT_SCHEMA_NAME_UNICODE_CLEAN = "false";
+
+ /**
* The primary key was originally written using a Unicode sort key.
* However, the JDK generates Unicode sort keys with embedded nuls and
* that broke the logic to detect the end of the Unicode primary keys.
@@ -1083,6 +1107,17 @@
* This is a global option since it was always <code>false</code> for
* historical stores.
*
+ * @see Options#SCHEMA_NAME_UNICODE_CLEAN
+ */
+ final static transient boolean schemaNameUnicodeClean = Boolean
+ .valueOf(System.getProperty(
+ SparseRowStore.Options.SCHEMA_NAME_UNICODE_CLEAN,
+ SparseRowStore.Options.DEFAULT_SCHEMA_NAME_UNICODE_CLEAN));
+
+ /**
+ * This is a global option since it was always <code>false</code> for
+ * historical stores.
+ *
* @see Options#PRIMARY_KEY_UNICODE_CLEAN
*/
final static transient boolean primaryKeyUnicodeClean = Boolean
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/cache/TestAll.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/cache/TestAll.java 2010-07-30 13:51:36 UTC (rev 3371)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/cache/TestAll.java 2010-07-30 13:51:43 UTC (rev 3372)
@@ -76,7 +76,8 @@
suite.addTestSuite(TestStoreAndAddressLRUCache.class);
- suite.addTestSuite(TestHardReferenceGlobalLRU.class);
+ // Note: This implementation is not used.
+// suite.addTestSuite(TestHardReferenceGlobalLRU.class);
suite.addTestSuite(TestHardReferenceGlobalLRURecycler.class);
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestConcurrentJournal.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestConcurrentJournal.java 2010-07-30 13:51:36 UTC (rev 3371)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestConcurrentJournal.java 2010-07-30 13:51:43 UTC (rev 3372)
@@ -971,190 +971,193 @@
}
- /**
- * Test verifies that an {@link ITx#UNISOLATED} task failure does not cause
- * concurrent writers to abort. The test also verifies that the
- * {@link Checkpoint} record for the named index is NOT updated since none
- * of the tasks write anything on the index.
- *
- * @todo The assumptions for this test may have been invalidated by the
- * recent (4/29) changes to the group commit and task commit protocol
- * and this test might need to be reworked or rewritten.
+ /*
+ * @todo revisit this unit test. It's semantics appear to have aged.
*/
- public void test_writeService001() throws Exception {
-
- final Journal journal = new Journal(getProperties());
+// /**
+// * Test verifies that an {@link ITx#UNISOLATED} task failure does not cause
+// * concurrent writers to abort. The test also verifies that the
+// * {@link Checkpoint} record for the named index is NOT updated since none
+// * of the tasks write anything on the index.
+// *
+// * @todo The assumptions for this test may have been invalidated by the
+// * recent (4/29) changes to the group commit and task commit protocol
+// * and this test might need to be reworked or rewritten.
+// */
+// public void test_writeService001() throws Exception {
+//
+// final Journal journal = new Journal(getProperties());
+//
+// try {
+//
+// final String name = "test";
+//
+// // Note: checkpoint for the newly registered index.
+// final long checkpointAddr0;
+// {
+//
+// journal.registerIndex(name,new IndexMetadata(name,UUID.randomUUID()));
+//
+// journal.commit();
+//
+// checkpointAddr0 = journal.getIndex(name).getCheckpoint()
+// .getCheckpointAddr();
+//
+// }
+//
+// // the list of tasks to be run.
+// final List<AbstractTask<Object>> tasks = new LinkedList<AbstractTask<Object>>();
+//
+// // NOP
+// tasks.add(new AbstractTask(journal, ITx.UNISOLATED, name) {
+// protected String getTaskName() {
+// return "a";
+// }
+// protected Object doTask() throws Exception {
+// assertEquals(checkpointAddr0, ((BTree) getIndex(name))
+// .getCheckpoint().getCheckpointAddr());
+// return null;
+// }
+// });
+//
+// // throws exception.
+// tasks.add(new AbstractTask(journal, ITx.UNISOLATED, name) {
+// protected String getTaskName() {
+// return "b";
+// }
+// protected Object doTask() throws Exception {
+// assertEquals(checkpointAddr0, ((BTree) getIndex(name))
+// .getCheckpoint().getCheckpointAddr());
+// throw new ForcedAbortException();
+// }
+// });
+//
+// // NOP
+// tasks.add(new AbstractTask(journal, ITx.UNISOLATED, name) {
+// protected String getTaskName() {
+// return "c";
+// }
+// protected Object doTask() throws Exception {
+// assertEquals(checkpointAddr0, ((BTree) getIndex(name))
+// .getCheckpoint().getCheckpointAddr());
+// return null;
+// }
+// });
+//
+// // the commit counter before we submit the tasks.
+// final long commitCounter0 = journal.getRootBlockView()
+// .getCommitCounter();
+//
+// // the write service on which the tasks execute.
+// final WriteExecutorService writeService = journal
+// .getConcurrencyManager().getWriteService();
+//
+// // the group commit count before we submit the tasks.
+// final long groupCommitCount0 = writeService.getGroupCommitCount();
+//
+// // the abort count before we submit the tasks.
+// final long abortCount0 = writeService.getAbortCount();
+//
+// // the #of failed tasks before we submit the tasks.
+// final long failedTaskCount0 = writeService.getTaskFailedCount();
+//
+// // the #of successfully tasks before we submit the tasks.
+// final long successTaskCount0 = writeService.getTaskSuccessCount();
+//
+// // the #of successfully committed tasks before we submit the tasks.
+// final long committedTaskCount0 = writeService.getTaskCommittedCount();
+//
+// // submit the tasks and await their completion.
+// final List<Future<Object>> futures = journal.invokeAll( tasks );
+//
+// /*
+// * verify the #of commits on the journal is unchanged since nothing
+// * is written by any of these tasks.
+// *
+// * The expectation is that the tasks that succeed make it into the
+// * same commit group while the task that throws an exception does
+// * not cause the commit group to be aborted.
+// *
+// * Note: The tasks will make it into the same commit group iff the
+// * first task that completes is willing to wait for the others to
+// * join the commit group.
+// *
+// * Note: The tasks have a dependency on the same resource so they
+// * will be serialized (executed in a strict sequence).
+// */
+// assertEquals("commitCounter", commitCounter0, journal
+// .getRootBlockView().getCommitCounter());
+//
+// // however, a group commit SHOULD have been performed.
+// assertEquals("groupCommitCount", groupCommitCount0 + 1, writeService
+// .getGroupCommitCount());
+//
+// // NO aborts should have been performed.
+// assertEquals("aboutCount", abortCount0, writeService.getAbortCount());
+//
+// // ONE(1) tasks SHOULD have failed.
+// assertEquals("failedTaskCount", failedTaskCount0 + 1, writeService.
+// getTaskFailedCount());
+//
+// // TWO(2) tasks SHOULD have succeeded.
+// assertEquals("successTaskCount", successTaskCount0 + 2, writeService
+// .getTaskSuccessCount());
+//
+// // TWO(2) successfull tasks SHOULD have been committed.
+// assertEquals("committedTaskCount", committedTaskCount0 + 2, writeService
+// .getTaskCommittedCount());
+//
+// assertEquals( 3, futures.size());
+//
+// // tasks[0]
+// {
+//
+// Future f = futures.get(0);
+//
+// assertTrue(f.isDone());
+//
+// f.get(); // No exception expected.
+//
+// }
+//
+// // tasks[2]
+// {
+//
+// Future f = futures.get(2);
+//
+// assertTrue(f.isDone());
+//
+// f.get(); // No exception expected.
+//
+// }
+//
+// // tasks[1]
+// {
+//
+// Future f = futures.get(1);
+//
+// assertTrue(f.isDone());
+//
+// try {
+// f.get();
+// fail("Expecting exception");
+// } catch(ExecutionException ex) {
+// assertTrue(InnerCause.isInnerCause(ex, ForcedAbortException.class));
+// }
+//
+// }
+//
+// assertEquals(checkpointAddr0, journal.getIndex(name)
+// .getCheckpoint().getCheckpointAddr());
+//
+// } finally {
+//
+// journal.destroy();
+//
+// }
+//
+// }
- try {
-
- final String name = "test";
-
- // Note: checkpoint for the newly registered index.
- final long checkpointAddr0;
- {
-
- journal.registerIndex(name,new IndexMetadata(name,UUID.randomUUID()));
-
- journal.commit();
-
- checkpointAddr0 = journal.getIndex(name).getCheckpoint()
- .getCheckpointAddr();
-
- }
-
- // the list of tasks to be run.
- final List<AbstractTask<Object>> tasks = new LinkedList<AbstractTask<Object>>();
-
- // NOP
- tasks.add(new AbstractTask(journal, ITx.UNISOLATED, name) {
- protected String getTaskName() {
- return "a";
- }
- protected Object doTask() throws Exception {
- assertEquals(checkpointAddr0, ((BTree) getIndex(name))
- .getCheckpoint().getCheckpointAddr());
- return null;
- }
- });
-
- // throws exception.
- tasks.add(new AbstractTask(journal, ITx.UNISOLATED, name) {
- protected String getTaskName() {
- return "b";
- }
- protected Object doTask() throws Exception {
- assertEquals(checkpointAddr0, ((BTree) getIndex(name))
- .getCheckpoint().getCheckpointAddr());
- throw new ForcedAbortException();
- }
- });
-
- // NOP
- tasks.add(new AbstractTask(journal, ITx.UNISOLATED, name) {
- protected String getTaskName() {
- return "c";
- }
- protected Object doTask() throws Exception {
- assertEquals(checkpointAddr0, ((BTree) getIndex(name))
- .getCheckpoint().getCheckpointAddr());
- return null;
- }
- });
-
- // the commit counter before we submit the tasks.
- final long commitCounter0 = journal.getRootBlockView()
- .getCommitCounter();
-
- // the write service on which the tasks execute.
- final WriteExecutorService writeService = journal
- .getConcurrencyManager().getWriteService();
-
- // the group commit count before we submit the tasks.
- final long groupCommitCount0 = writeService.getGroupCommitCount();
-
- // the abort count before we submit the tasks.
- final long abortCount0 = writeService.getAbortCount();
-
- // the #of failed tasks before we submit the tasks.
- final long failedTaskCount0 = writeService.getTaskFailedCount();
-
- // the #of successfully tasks before we submit the tasks.
- final long successTaskCount0 = writeService.getTaskSuccessCount();
-
- // the #of successfully committed tasks before we submit the tasks.
- final long committedTaskCount0 = writeService.getTaskCommittedCount();
-
- // submit the tasks and await their completion.
- final List<Future<Object>> futures = journal.invokeAll( tasks );
-
- /*
- * verify the #of commits on the journal is unchanged since nothing
- * is written by any of these tasks.
- *
- * The expectation is that the tasks that succeed make it into the
- * same commit group while the task that throws an exception does
- * not cause the commit group to be aborted.
- *
- * Note: The tasks will make it into the same commit group iff the
- * first task that completes is willing to wait for the others to
- * join the commit group.
- *
- * Note: The tasks have a dependency on the same resource so they
- * will be serialized (executed in a strict sequence).
- */
- assertEquals("commitCounter", commitCounter0, journal
- .getRootBlockView().getCommitCounter());
-
- // however, a group commit SHOULD have been performed.
- assertEquals("groupCommitCount", groupCommitCount0 + 1, writeService
- .getGroupCommitCount());
-
- // NO aborts should have been performed.
- assertEquals("aboutCount", abortCount0, writeService.getAbortCount());
-
- // ONE(1) tasks SHOULD have failed.
- assertEquals("failedTaskCount", failedTaskCount0 + 1, writeService.
- getTaskFailedCount());
-
- // TWO(2) tasks SHOULD have succeeded.
- assertEquals("successTaskCount", successTaskCount0 + 2, writeService
- .getTaskSuccessCount());
-
- // TWO(2) successfull tasks SHOULD have been committed.
- assertEquals("committedTaskCount", committedTaskCount0 + 2, writeService
- .getTaskCommittedCount());
-
- assertEquals( 3, futures.size());
-
- // tasks[0]
- {
-
- Future f = futures.get(0);
-
- assertTrue(f.isDone());
-
- f.get(); // No exception expected.
-
- }
-
- // tasks[2]
- {
-
- Future f = futures.get(2);
-
- assertTrue(f.isDone());
-
- f.get(); // No exception expected.
-
- }
-
- // tasks[1]
- {
-
- Future f = futures.get(1);
-
- assertTrue(f.isDone());
-
- try {
- f.get();
- fail("Expecting exception");
- } catch(ExecutionException ex) {
- assertTrue(InnerCause.isInnerCause(ex, ForcedAbortException.class));
- }
-
- }
-
- assertEquals(checkpointAddr0, journal.getIndex(name)
- ...
[truncated message content] |
|
From: <btm...@us...> - 2010-07-30 20:41:41
|
Revision: 3377
http://bigdata.svn.sourceforge.net/bigdata/?rev=3377&view=rev
Author: btmurphy
Date: 2010-07-30 20:41:35 +0000 (Fri, 30 Jul 2010)
Log Message:
-----------
[branch dev-btm]: fixed a typo in JiniServicesHelper that prevented TestBigdataClient from passing when run on eclipse
Modified Paths:
--------------
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/JiniServicesHelper.java
branches/dev-btm/bigdata-jini/src/resources/config/bigdataStandaloneTesting.config
branches/dev-btm/src/resources/config/bigdataCluster.config
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/JiniServicesHelper.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/JiniServicesHelper.java 2010-07-30 18:49:37 UTC (rev 3376)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/JiniServicesHelper.java 2010-07-30 20:41:35 UTC (rev 3377)
@@ -586,7 +586,7 @@
optionsList.add(options[i]);
}
String joinGroupsOverrideStr =
- "com.bigdata.loadblancer.groupsToJoin=new String[] "
+ "com.bigdata.loadbalancer.groupsToJoin=new String[] "
+"{"
+"\""+fedname+"\""
+"}";
Modified: branches/dev-btm/bigdata-jini/src/resources/config/bigdataStandaloneTesting.config
===================================================================
--- branches/dev-btm/bigdata-jini/src/resources/config/bigdataStandaloneTesting.config 2010-07-30 18:49:37 UTC (rev 3376)
+++ branches/dev-btm/bigdata-jini/src/resources/config/bigdataStandaloneTesting.config 2010-07-30 20:41:35 UTC (rev 3377)
@@ -36,6 +36,14 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeUnit.*;
+import java.net.NetworkInterface;
+import net.jini.constraint.BasicMethodConstraints;
+import net.jini.core.constraint.ConnectionRelativeTime;
+import net.jini.core.constraint.InvocationConstraints;
+import net.jini.discovery.LookupDiscoveryManager;
+import com.bigdata.util.config.NicUtil;
+import com.bigdata.util.config.ConfigDeployUtil;
+
/*
* This is a standalone configuration file for a bigdata federation
* used for unit tests in conjunction with the JiniServicesHelper class.
@@ -431,6 +439,61 @@
}
+com.bigdata.loadbalancer {
+
+ private static exportIpAddr =
+ NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
+ private static exportPort =
+ Integer.parseInt( System.getProperty("exportPort", "0") );
+
+ groupsToJoin = ConfigDeployUtil.getGroupsToDiscover();
+ locatorsToJoin = ConfigDeployUtil.getLocatorsToDiscover();
+
+ private static exporterTcpServerEndpoint =
+ TcpServerEndpoint.getInstance(exportIpAddr, exportPort);
+ private static serverILFactory =
+ new BasicILFactory(
+ new BasicMethodConstraints(
+ new InvocationConstraints(
+ new ConnectionRelativeTime(
+ ConfigDeployUtil.getLong(
+ "rmi.connectTimeout")),
+ null)),
+ null);
+
+ serverExporter =
+ new BasicJeriExporter
+ (TcpServerEndpoint.getInstance(exportIpAddr,exportPort),
+ serverILFactory,
+ false,
+ true);
+
+ discoveryManager = new LookupDiscoveryManager(groupsToJoin,
+ locatorsToJoin,
+ null,
+ this);
+ // Where service state is persisted
+ persistenceDirectory =
+ ConfigUtil.concat
+ ( new String[] { System.getProperty("app.home", "${user.dir}"),
+ "${/}var${/}state${/}loadbalancerState" } );
+}
+
+net.jini.discovery.LookupDiscovery {
+ multicastRequestHost = com.bigdata.loadbalancer.exportIpAddr;
+ multicastInterfaces = new NetworkInterface[] {
+ NicUtil.getNetworkInterface(com.bigdata.loadbalancer.exportIpAddr)
+ };
+}
+
+net.jini.lookup.ServiceDiscoveryManager {
+ eventListenerExporter =
+ new BasicJeriExporter
+ (com.bigdata.loadbalancer.exporterTcpServerEndpoint,
+ com.bigdata.loadbalancer.serverILFactory, false, false);
+}
+
+
/*
* Note: Whitespace intentionally follows this comment!
*/
Modified: branches/dev-btm/src/resources/config/bigdataCluster.config
===================================================================
--- branches/dev-btm/src/resources/config/bigdataCluster.config 2010-07-30 18:49:37 UTC (rev 3376)
+++ branches/dev-btm/src/resources/config/bigdataCluster.config 2010-07-30 20:41:35 UTC (rev 3377)
@@ -1201,7 +1201,6 @@
"-Dlog4j.primary.configuration=@NAS@/dist/bigdata/var/config/logging/loadbalancer-logging.properties",
"-DusingServicesManagerService=true",
"-Dbigdata.logDir=@NAS@/dist/bigdata/var/log",
- "-DappHome=@NAS@/dist/bigdata",
"-DappHome=@APP_HOME@",
"-Dconfig=@NAS@/dist/bigdata/var/config/jini/loadbalancer.config"
};
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <btm...@us...> - 2010-07-30 22:46:27
|
Revision: 3380
http://bigdata.svn.sourceforge.net/bigdata/?rev=3380&view=rev
Author: btmurphy
Date: 2010-07-30 22:46:19 +0000 (Fri, 30 Jul 2010)
Log Message:
-----------
merge -r:3370:HEAD(3378) ~/bigdata/trunk ~/bigdata/branches/dev-btm [trunk --> branch dev-btm]
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/btree/BigdataMap.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/IKeyBuilder.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractBTreeTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractTupleCursorTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestBTreeLeafCursors.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestBigdataMap.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestChunkedIterators.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestCopyOnWrite.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestDirtyIterators.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIncrementalWrite.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexSegmentBuilderCacheInteraction.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexSegmentBuilderWithCompactingMerge.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexSegmentBuilderWithIncrementalBuild.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexSegmentBuilderWithSmallTree.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexSegmentWithBloomFilter.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestInsertLookupRemoveKeysInRootLeaf.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIterators.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestLinearListMethods.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestMutableBTreeCursors.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestReopen.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestSplitJoinRootLeaf.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestSplitJoinThreeLevels.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestSplitRootLeaf.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestTouch.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestTransientBTree.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/filter/TestTupleFilters.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/AbstractUnicodeKeyBuilderTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestSuccessorUtil.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/raba/codec/AbstractRabaCoderTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/raba/codec/RandomURIGenerator.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestBuildTask.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestBuildTask2.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestMergeTask.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestOverflow.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestResourceManagerBootstrap.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestSegSplitter.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestMove.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestRangeQuery.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestRestartSafe.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestScatterSplit.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestSplitJoin.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/ndx/pipeline/TestMasterTaskWithSplits.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/TestBigdataClient.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/internal/BlobOverflowHandler.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPO.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPORelation.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOTupleSerializer.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOValueCoders.java
Property Changed:
----------------
branches/dev-btm/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/attr/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/disco/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/
branches/dev-btm/bigdata-perf/
branches/dev-btm/bigdata-perf/lubm/lib/
branches/dev-btm/bigdata-perf/lubm/src/resources/
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util/
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/
branches/dev-btm/dsi-utils/LEGAL/
branches/dev-btm/dsi-utils/lib/
branches/dev-btm/dsi-utils/src/
branches/dev-btm/dsi-utils/src/test/
branches/dev-btm/dsi-utils/src/test/it/
branches/dev-btm/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/osgi/
Property changes on: branches/dev-btm
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/LEXICON_REFACTOR_BRANCH:2633-3304
/branches/bugfix-btm:2594-3237
/branches/fko:3150-3194
/trunk:2575-2594,2596-2877,2882-2903,2910-3370
+ /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/LEXICON_REFACTOR_BRANCH:2633-3304
/branches/bugfix-btm:2594-3237
/branches/fko:3150-3194
/trunk:2575-2594,2596-2877,2882-2903,2910-3378
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/BigdataMap.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/BigdataMap.java 2010-07-30 22:14:17 UTC (rev 3379)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/BigdataMap.java 2010-07-30 22:46:19 UTC (rev 3380)
@@ -58,21 +58,21 @@
* Note: The total order of the {@link BigdataMap} is completely determined by
* {@link ITupleSerializer#serializeKey(Object)}. There is NO concept of a
* {@link Comparator}. The {@link ITupleSerializer} is responsible for coercing
- * application keys into variable length <strong>unsigned</strong> byte[]s
- * which are the keys for the underlying B+Tree. The order for the B+Tree is the
+ * application keys into variable length <strong>unsigned</strong> byte[]s which
+ * are the keys for the underlying B+Tree. The order for the B+Tree is the
* natural order for the <strong>unsigned byte[]</strong>s. {@link KeyBuilder}
* supports the generation of unsigned byte[]s from various kinds of Java
* primitives and Unicode {@link String}s and is typically used to write the
* {@link ITupleSerializer#serializeKey(Object)} method.
* <p>
- * Note: The coercion of the application keys into unsigned byte[]s is not
- * typesafe unless you either consistently use a strongly typed instance of this
+ * Note: The coercion of the application keys into unsigned byte[]s is not type
+ * safe unless you either consistently use a strongly typed instance of this
* class or specify an {@link ITupleSerializer} for the backing B+Tree that only
* allows application keys that are instances of acceptable classes. This issue
* is more critical for keys than for values since the keys define the total
* index order and the default coercion rules for keys are provided by
- * {@link KeyBuilder#asSortKey(Object)} which does not attenpt to partition the
- * key space by the application key type (keys are not safely polymorphic by
+ * {@link KeyBuilder#append(Object)} which does not attempt to partition the key
+ * space by the application key type (keys are not safely polymorphic by
* default).
* <p>
* Note: When storing Java objects in the tuple value, the value MUST be treated
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/IKeyBuilder.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/IKeyBuilder.java 2010-07-30 22:14:17 UTC (rev 3379)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/IKeyBuilder.java 2010-07-30 22:46:19 UTC (rev 3380)
@@ -33,6 +33,7 @@
import java.util.Locale;
import java.util.Properties;
import java.util.UUID;
+
import com.bigdata.btree.BytesUtil;
import com.bigdata.btree.keys.KeyBuilder.Options;
@@ -88,7 +89,6 @@
* {@link #appendText(String, boolean, boolean)}.
* </p>
*
- * @see KeyBuilder#asSortKey(Object)
* @see KeyBuilder#newInstance()
* @see KeyBuilder#newUnicodeInstance()
* @see KeyBuilder#newUnicodeInstance(Properties)
@@ -97,7 +97,7 @@
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
*/
-public interface IKeyBuilder extends ISortKeyBuilder {
+public interface IKeyBuilder extends ISortKeyBuilder<Object> {
/**
* The #of bytes of data in the key.
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-07-30 22:14:17 UTC (rev 3379)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-07-30 22:46:19 UTC (rev 3380)
@@ -1065,78 +1065,8 @@
}
- /*
- * static helper methods.
- */
-
- /**
- * Used to unbox an application key (convert it to an unsigned byte[]).
- */
- static private final IKeyBuilder _keyBuilder = newUnicodeInstance();
-
- /**
- * Utility method converts an application key to a sort key (an unsigned
- * byte[] that imposes the same sort order).
- * <p>
- * Note: This method is thread-safe.
- * <p>
- * Note: Strings are Unicode safe for the default locale. See
- * {@link Locale#getDefault()}. If you require a specific local or different
- * locals at different times or for different indices then you MUST
- * provision and apply your own {@link KeyBuilder}.
- *
- * @param val
- * An application key.
- *
- * @return The unsigned byte[] equivalent of that key. This will be
- * <code>null</code> iff the <i>key</i> is <code>null</code>. If the
- * <i>key</i> is a byte[], then the byte[] itself will be returned.
- *
- * @deprecated This method circumvents explicit configuration of the
- * {@link KeyBuilder} and is used nearly exclusively by unit
- * tests. While explicit configuration is not required for keys
- * which do not include Unicode sort key components, this method
- * also relies on a single global {@link KeyBuilder} instance
- * protected by a lock. That lock is therefore a bottleneck. The
- * correct practice is to use thread-local or per task
- * {@link IKeyBuilder}s to avoid lock contention.
- */
- @SuppressWarnings("unchecked")
- public static final byte[] asSortKey(Object val) {
+ public byte[] getSortKey(final Object val) {
- if (val == null) {
-
- return null;
-
- }
-
- if (val instanceof byte[]) {
-
- return (byte[]) val;
-
- }
-
- /*
- * Synchronize on the keyBuilder to avoid concurrent modification of its
- * state.
- */
-
- synchronized (_keyBuilder) {
-
- return _keyBuilder.getSortKey(val);
-
-// _keyBuilder.reset();
-//
-// _keyBuilder.append( key );
-//
-// return _keyBuilder.getKey();
-
- }
-
- }
-
- public byte[] getSortKey(Object val) {
-
reset();
append( val );
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java 2010-07-30 22:14:17 UTC (rev 3379)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java 2010-07-30 22:46:19 UTC (rev 3380)
@@ -141,7 +141,7 @@
/*
* One time encoding of the schema name as a Unicode sort key.
*/
- schemaBytes = KeyBuilder.asSortKey(name);
+ schemaBytes = asSortKey(name);
}
}
@@ -501,5 +501,51 @@
+ ",primaryKeyType=" + getPrimaryKeyType() + "}";
}
+
+ /**
+ * Used for historical compatibility to unbox an application key (convert it
+ * to an unsigned byte[]).
+ */
+ static private final IKeyBuilder _keyBuilder = KeyBuilder.newUnicodeInstance();
+
+ /**
+ * Utility method for historical compatibility converts an application key
+ * to a sort key (an unsigned byte[] that imposes the same sort order).
+ * <p>
+ * Note: This method is thread-safe.
+ *
+ * @param val
+ * An application key.
+ *
+ * @return The unsigned byte[] equivalent of that key. This will be
+ * <code>null</code> iff the <i>key</i> is <code>null</code>. If the
+ * <i>key</i> is a byte[], then the byte[] itself will be returned.
+ */
+ private static final byte[] asSortKey(final Object val) {
+
+ if (val == null) {
+
+ return null;
+
+ }
+
+ if (val instanceof byte[]) {
+
+ return (byte[]) val;
+
+ }
+
+ /*
+ * Synchronize on the keyBuilder to avoid concurrent modification of its
+ * state.
+ */
+
+ synchronized (_keyBuilder) {
+
+ return _keyBuilder.getSortKey(val);
+
+ }
+ }
+
}
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractBTreeTestCase.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractBTreeTestCase.java 2010-07-30 22:14:17 UTC (rev 3379)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractBTreeTestCase.java 2010-07-30 22:46:19 UTC (rev 3380)
@@ -49,6 +49,7 @@
import com.bigdata.btree.keys.IKeyBuilder;
import com.bigdata.btree.keys.KV;
import com.bigdata.btree.keys.KeyBuilder;
+import com.bigdata.btree.keys.TestKeyBuilder;
import com.bigdata.btree.raba.IRaba;
import com.bigdata.btree.raba.codec.RandomKeysGenerator;
import com.bigdata.cache.HardReferenceQueue;
@@ -1097,7 +1098,7 @@
assertEquals("#entries",i,btree.nentries);
- final byte[] key = KeyBuilder.asSortKey(ikey);
+ final byte[] key = TestKeyBuilder.asSortKey(ikey);
assertNull(btree.lookup(key));
@@ -1133,7 +1134,7 @@
for( int i=0; i<keys.length; i++ ) {
- byte[] key = KeyBuilder.asSortKey(keys[i]);
+ byte[] key = TestKeyBuilder.asSortKey(keys[i]);
assertEquals(entries[i],btree.lookup(key));
assertEquals(entries[i],btree.remove(key));
@@ -1216,7 +1217,7 @@
assertEquals("#entries",i,btree.nentries);
- final byte[] key = KeyBuilder.asSortKey(ikey);
+ final byte[] key = TestKeyBuilder.asSortKey(ikey);
assertNull(btree.lookup(key));
@@ -1252,7 +1253,7 @@
for( int i=0; i<keys.length; i++ ) {
- final byte[] key = KeyBuilder.asSortKey(keys[i]);
+ final byte[] key = TestKeyBuilder.asSortKey(keys[i]);
assertEquals(entries[i],btree.lookup(key));
assertEquals(entries[i],btree.remove(key));
@@ -1497,7 +1498,7 @@
assertEquals("#entries", i, btree.nentries);
- final byte[] key = KeyBuilder.asSortKey(ikey);
+ final byte[] key = TestKeyBuilder.asSortKey(ikey);
assertNull(btree.lookup(key));
@@ -1687,7 +1688,7 @@
assertEquals("#entries",i,btree.nentries);
- final byte[] key = KeyBuilder.asSortKey(ikey);
+ final byte[] key = TestKeyBuilder.asSortKey(ikey);
assertNull(btree.lookup(key));
@@ -1774,7 +1775,7 @@
final Integer ikey = keys[index];
- final byte[] key = KeyBuilder.asSortKey(ikey);
+ final byte[] key = TestKeyBuilder.asSortKey(ikey);
final SimpleEntry val = vals[index];
@@ -1816,7 +1817,7 @@
Map.Entry<Integer,SimpleEntry> entry = itr.next();
- final byte[] tmp = KeyBuilder.asSortKey(entry.getKey());
+ final byte[] tmp = TestKeyBuilder.asSortKey(entry.getKey());
assertEquals("lookup(" + entry.getKey() + ")", entry
.getValue(), btree.lookup(tmp));
@@ -1855,7 +1856,7 @@
for( int i=0; i<nkeys; i++ ) {
- keys[i] = KeyBuilder.asSortKey(i+1); // Note: this produces dense keys with origin ONE(1).
+ keys[i] = TestKeyBuilder.asSortKey(i+1); // Note: this produces dense keys with origin ONE(1).
vals[i] = new SimpleEntry();
@@ -2597,7 +2598,7 @@
for (int i = 0; i < N; i++) {
// @todo param governs chance of a key collision and maximum #of distinct keys.
- final byte[] key = KeyBuilder.asSortKey(r.nextInt(100000));
+ final byte[] key = TestKeyBuilder.asSortKey(r.nextInt(100000));
// Note: #of bytes effects very little that we want to test so we keep it small.
final byte[] val = new byte[4];
Modified: branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractTupleCursorTestCase.java
===================================================================
--- branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractTupleCursorTestCase.java 2010-07-30 22:14:17 UTC (rev 3379)
+++ branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractTupleCursorTestCase.java 2010-07-30 22:46:19 UTC (rev 3380)
@@ -33,7 +33,7 @@
import junit.framework.TestCase2;
-import com.bigdata.btree.keys.KeyBuilder;
+import com.bigdata.btree.keys.TestKeyBuilder;
import com.bigdata.btree.raba.ReadOnlyKeysRaba;
import com.bigdata.rawstore.SimpleMemoryRawStore;
@@ -287,65 +287,65 @@
// seek to a probe key that does not exist.
assertEquals(null, cursor.seek(29));
assertEquals(null, cursor.tuple());
- assertEquals(KeyBuilder.asSortKey(29),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(29),cursor.currentKey());
assertTrue(cursor.hasNext());
assertEquals(new TestTuple<String>(30, "James"), cursor.next());
assertEquals(new TestTuple<String>(30, "James"), cursor.tuple());
- assertEquals(KeyBuilder.asSortKey(30),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(30),cursor.currentKey());
assertFalse(cursor.hasNext());
assertTrue(cursor.hasPrior());
assertEquals(new TestTuple<String>(20, "Mike"), cursor.prior());
assertEquals(new TestTuple<String>(20, "Mike"), cursor.tuple());
- assertEquals(KeyBuilder.asSortKey(20),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(20),cursor.currentKey());
// seek to a probe key that does not exist.
assertEquals(null, cursor.seek(9));
assertEquals(null, cursor.tuple());
- assertEquals(KeyBuilder.asSortKey(9),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(9),cursor.currentKey());
assertTrue(cursor.hasNext());
assertEquals(new TestTuple<String>(10, "Bryan"), cursor.next());
assertEquals(new TestTuple<String>(10, "Bryan"), cursor.tuple());
- assertEquals(KeyBuilder.asSortKey(10),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(10),cursor.currentKey());
assertFalse(cursor.hasPrior());
assertTrue(cursor.hasNext());
assertEquals(new TestTuple<String>(20, "Mike"), cursor.next());
- assertEquals(KeyBuilder.asSortKey(20),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(20),cursor.currentKey());
// seek to a probe key that does not exist and scan forward.
assertEquals(null, cursor.seek(19));
assertEquals(null, cursor.tuple());
- assertEquals(KeyBuilder.asSortKey(19),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(19),cursor.currentKey());
assertTrue(cursor.hasNext());
assertEquals(new TestTuple<String>(20, "Mike"), cursor.next());
assertEquals(new TestTuple<String>(20, "Mike"), cursor.tuple());
- assertEquals(KeyBuilder.asSortKey(20),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(20),cursor.currentKey());
assertTrue(cursor.hasNext());
assertEquals(new TestTuple<String>(30, "James"), cursor.next());
assertEquals(new TestTuple<String>(30, "James"), cursor.tuple());
- assertEquals(KeyBuilder.asSortKey(30),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(30),cursor.currentKey());
// seek to a probe key that does not exist and scan backward.
assertEquals(null, cursor.seek(19));
assertEquals(null, cursor.tuple());
- assertEquals(KeyBuilder.asSortKey(19),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(19),cursor.currentKey());
assertTrue(cursor.hasPrior());
assertEquals(new TestTuple<String>(10, "Bryan"), cursor.prior());
assertEquals(new TestTuple<String>(10, "Bryan"), cursor.tuple());
- assertEquals(KeyBuilder.asSortKey(10),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(10),cursor.currentKey());
assertFalse(cursor.hasPrior());
// seek to a probe key that does not exist (after all valid tuples).
assertEquals(null, cursor.seek(31));
assertEquals(null, cursor.tuple());
assertTrue(cursor.isCursorPositionDefined());
- assertEquals(KeyBuilder.asSortKey(31),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(31),cursor.currentKey());
assertFalse(cursor.hasNext());
// seek to a probe key that does not exist (after all valid tuples).
assertEquals(null, cursor.seek(31));
assertEquals(null, cursor.tuple());
assertTrue(cursor.isCursorPositionDefined());
- assertEquals(KeyBuilder.asSortKey(31),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(31),cursor.currentKey());
assertTrue(cursor.hasPrior());
assertEquals(new TestTuple<String>(30, "James"), cursor.prior());
@@ -369,9 +369,9 @@
*/
{
- final byte[] fromKey = KeyBuilder.asSortKey(10);
+ final byte[] fromKey = TestKeyBuilder.asSortKey(10);
- final byte[] toKey = KeyBuilder.asSortKey(20);
+ final byte[] toKey = TestKeyBuilder.asSortKey(20);
final ITupleCursor2<String> cursor = newCursor(btree,
IRangeQuery.DEFAULT, fromKey, toKey);
@@ -412,9 +412,9 @@
*/
{
- final byte[] fromKey = KeyBuilder.asSortKey(20);
+ final byte[] fromKey = TestKeyBuilder.asSortKey(20);
- final byte[] toKey = KeyBuilder.asSortKey(30);
+ final byte[] toKey = TestKeyBuilder.asSortKey(30);
final ITupleCursor2<String> cursor = newCursor(btree,
IRangeQuery.DEFAULT, fromKey, toKey);
@@ -458,9 +458,9 @@
*/
{
- final byte[] fromKey = KeyBuilder.asSortKey(10);
+ final byte[] fromKey = TestKeyBuilder.asSortKey(10);
- final byte[] toKey = KeyBuilder.asSortKey(19);
+ final byte[] toKey = TestKeyBuilder.asSortKey(19);
final ITupleCursor2<String> cursor = newCursor(btree,
IRangeQuery.DEFAULT, fromKey, toKey);
@@ -471,7 +471,7 @@
// assertEquals(KeyBuilder.asSortKey(19),cursor.currentKey());
assertTrue(cursor.hasPrior());
assertEquals(new TestTuple<String>(10, "Bryan"), cursor.prior());
- assertEquals(KeyBuilder.asSortKey(10),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(10),cursor.currentKey());
assertFalse(cursor.hasPrior());
}
@@ -481,19 +481,19 @@
*/
{
- final byte[] fromKey = KeyBuilder.asSortKey(10);
+ final byte[] fromKey = TestKeyBuilder.asSortKey(10);
- final byte[] toKey = KeyBuilder.asSortKey(29);
+ final byte[] toKey = TestKeyBuilder.asSortKey(29);
final ITupleCursor2<String> cursor = newCursor(btree,
IRangeQuery.DEFAULT, fromKey, toKey);
assertTrue(cursor.hasPrior());
assertEquals(new TestTuple<String>(20, "Mike"), cursor.prior());
- assertEquals(KeyBuilder.asSortKey(20),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(20),cursor.currentKey());
assertTrue(cursor.hasPrior());
assertEquals(new TestTuple<String>(10, "Bryan"), cursor.prior());
- assertEquals(KeyBuilder.asSortKey(10),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(10),cursor.currentKey());
assertFalse(cursor.hasPrior());
}
@@ -503,16 +503,16 @@
*/
{
- final byte[] fromKey = KeyBuilder.asSortKey(10);
+ final byte[] fromKey = TestKeyBuilder.asSortKey(10);
- final byte[] toKey = KeyBuilder.asSortKey(11);
+ final byte[] toKey = TestKeyBuilder.asSortKey(11);
final ITupleCursor2<String> cursor = newCursor(btree,
IRangeQuery.DEFAULT, fromKey, toKey);
assertTrue(cursor.hasPrior());
assertEquals(new TestTuple<String>(10, "Bryan"), cursor.prior());
- assertEquals(KeyBuilder.asSortKey(10),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(10),cursor.currentKey());
assertFalse(cursor.hasPrior());
}
@@ -574,15 +574,15 @@
assertEquals("ntuples", 5, btree.getEntryCount());
// The separator key is (30).
- assertEquals(KeyBuilder.asSortKey(30), ((Node) btree.getRoot())
+ assertEquals(TestKeyBuilder.asSortKey(30), ((Node) btree.getRoot())
.getKeys().get(0));
// Verify the expected keys in the 1st leaf.
AbstractBTreeTestCase.assertKeys(
//
new ReadOnlyKeysRaba(new byte[][] {//
- KeyBuilder.asSortKey(10), //
- KeyBuilder.asSortKey(20), //
+ TestKeyBuilder.asSortKey(10), //
+ TestKeyBuilder.asSortKey(20), //
}),//
((Node) btree.getRoot()).getChild(0/* 1st leaf */).getKeys());
@@ -590,9 +590,9 @@
AbstractBTreeTestCase.assertKeys(
//
new ReadOnlyKeysRaba(new byte[][] {//
- KeyBuilder.asSortKey(30), //
- KeyBuilder.asSortKey(40), //
- KeyBuilder.asSortKey(50),//
+ TestKeyBuilder.asSortKey(30), //
+ TestKeyBuilder.asSortKey(40), //
+ TestKeyBuilder.asSortKey(50),//
}),//
((Node) btree.getRoot()).getChild(1/* 2nd leaf */).getKeys());
@@ -627,16 +627,16 @@
*/
{
- final byte[] fromKey = KeyBuilder.asSortKey(10);
+ final byte[] fromKey = TestKeyBuilder.asSortKey(10);
- final byte[] toKey = KeyBuilder.asSortKey(30);
+ final byte[] toKey = TestKeyBuilder.asSortKey(30);
final ITupleCursor2<String> cursor = newCursor(btree,
IRangeQuery.DEFAULT, fromKey, toKey);
assertTrue(cursor.hasPrior());
assertEquals(new TestTuple<String>(20, "Mike"), cursor.prior());
- assertEquals(KeyBuilder.asSortKey(20),cursor.currentKey());
+ assertEquals(TestKeyBuilder.asSortKey(20),cursor.currentKey());
}
@@ -647,16 +647,16 @@
*/
{
- final byte[] fromKey = KeyBuilder.asSortKey(0);
+ final byte[] fromKey = TestKeyBuilder.asSortKey(0);
- final byte[] toKey = KeyBuilder.asSortK...
[truncated message content] |
|
From: <btm...@us...> - 2010-08-06 16:51:54
|
Revision: 3426
http://bigdata.svn.sourceforge.net/bigdata/?rev=3426&view=rev
Author: btmurphy
Date: 2010-08-06 16:51:44 +0000 (Fri, 06 Aug 2010)
Log Message:
-----------
[branch dev-btm]: CHECKPOINT - changes to allow smart proxy implementation of load balancer to be started by ServicesManagerService or BootManager, new/changed tests to run against either the smart proxy implementation of the load balancer or the purely remote implemenation (LoadBalancerService & LoadBalancerService)
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/RuleStats.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/CompactingMergeTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/DefaultServiceFederationDelegate.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedClient.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/Event.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/HostScore.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IBigdataClient.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IBigdataFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IFederationDelegate.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ILoadBalancerService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IServiceLoadHelper.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/LoadBalancer.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ServiceScore.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/IAsynchronousWriteBufferFactory.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/search/TestAll.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/StressTestConcurrent.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestAll.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestEventReceiver.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestMove.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/ndx/TestAll.java
branches/dev-btm/bigdata/src/test/com/bigdata/sparse/TestAll.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/BigdataZooDefs.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServicesManagerServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/BigdataServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/LoadBalancerConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServicesManagerConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/ZookeeperProcessHelper.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/AdminProxy.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/EmbeddedLoadBalancer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/PrivateInterface.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/ServiceProxy.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/JiniFederation.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/benchmark/ThroughputMaster.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/lookup/AbstractCachingServiceClient.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/lookup/BigdataCachingServiceClient.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/lookup/LoadBalancerClient.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/BroadcastSighup.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/JiniServicesHelper.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/TestAll.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/TestBigdataClient.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/master/TestAll.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/master/TestMappedRDFDataLoadMaster.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/ConcurrentDataLoader.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/RDFLoadTaskFactory.java
branches/dev-btm/src/resources/config/bigdataCluster.config
Added Paths:
-----------
branches/dev-btm/bigdata/src/test/com/bigdata/service/StressTestConcurrentRemote.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestBasicIndexStuffRemote.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestDistributedTransactionServiceRemote.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestEDSRemote.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestEmbeddedClientRemote.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestMetadataIndexRemote.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestMoveRemote.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestOverflowRemote.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestRangeQueryRemote.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestRestartSafeRemote.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestScatterSplitRemote.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestSplitJoinRemote.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/TestBigdataClientRemote.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/master/TestMappedRDFDataLoadMasterRemote.java
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/RuleStats.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/RuleStats.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/RuleStats.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -40,7 +40,6 @@
import com.bigdata.relation.rule.ISlice;
import com.bigdata.relation.rule.IStep;
import com.bigdata.relation.rule.Rule;
-//BTM import com.bigdata.service.ILoadBalancerService;
import com.bigdata.striterator.IKeyOrder;
import com.bigdata.service.LoadBalancer;
@@ -60,8 +59,7 @@
* In order to aggregate the data on rule execution, we want to roll up the data
* for the individual rules along the same lines as the program structure.
*
-BTM * @todo Report as counters aggregated by the {@link ILoadBalancerService}?
-* @todo Report as counters aggregated by the {@link LoadBalancer} service?
+ * @todo Report as counters aggregated by the load balancer service?
*
* @author mikep
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -61,7 +61,6 @@
import com.bigdata.service.EventResource;
import com.bigdata.service.EventType;
import com.bigdata.service.IDataService;
-//BTM import com.bigdata.service.ILoadBalancerService;
import com.bigdata.service.MetadataService;
import com.bigdata.util.InnerCause;
import com.bigdata.util.concurrent.DaemonThreadFactory;
@@ -1327,19 +1326,15 @@
}
/**
-BTM * Return the {@link ILoadBalancerService} if it can be discovered.
-* Return the {@link LoadBalancer} if it can be discovered.
+ * Return the load balancer service if it can be discovered.
*
-BTM * @return the {@link ILoadBalancerService} if it can be discovered and
-* @return the {@link LoadBalancer} service if it can be discovered and
+ * @return the load balancer service if it can be discovered and
* otherwise <code>null</code>.
*/
-//BTM protected ILoadBalancerService getLoadBalancerService() {
-protected LoadBalancer getLoadBalancerService() {
+ protected LoadBalancer getLoadBalancerService() {
// lookup the load balancer service.
-//BTM final ILoadBalancerService loadBalancerService;
-final LoadBalancer loadBalancerService;
+ final LoadBalancer loadBalancerService;
try {
@@ -1377,8 +1372,7 @@
* @param loadBalancerService
* The load balancer.
*/
-//BTM protected boolean shouldMove(final ILoadBalancerService loadBalancerService) {
-protected boolean shouldMove(final LoadBalancer loadBalancerService) {
+ protected boolean shouldMove(final LoadBalancer loadBalancerService) {
if (loadBalancerService == null)
throw new IllegalArgumentException();
@@ -1465,8 +1459,7 @@
* @return The tasks.
*/
private List<AbstractTask> chooseMoves(
-final LoadBalancer loadBalancerService) {
-//BTM final ILoadBalancerService loadBalancerService) {
+ final LoadBalancer loadBalancerService) {
if (resourceManager.maximumMovesPerTarget == 0) {
@@ -2388,8 +2381,7 @@
* Running out of DISK space causes an urgent condition and can lead to
* failure or all services on the same host. Therefore, when a host is near
* to exhausting its DISK space it (a) MUST notify the
-BTM * {@link ILoadBalancerService}; (b) temporary files SHOULD be purged; it
-* {@link ILoadBalancerService}; (b) temporary files SHOULD be purged; it
+ * load balancer service; (b) temporary files SHOULD be purged; it
* MAY choose to shed indices that are "hot for write" since that will slow
* down the rate at which the disk space is consumed; (d) index partitions
* may be aggressively moved off of the LDS; (e) the transaction service MAY
@@ -2464,8 +2456,7 @@
* When it is not available we simply do not consider index
* partition moves.
*/
-//BTM final ILoadBalancerService lbs = getLoadBalancerService();
-final LoadBalancer lbs = getLoadBalancerService();
+ final LoadBalancer lbs = getLoadBalancerService();
if(lbs != null && shouldMove(lbs)) {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/resources/CompactingMergeTask.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/resources/CompactingMergeTask.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/resources/CompactingMergeTask.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -29,7 +29,6 @@
import com.bigdata.service.DataService;
import com.bigdata.service.Event;
import com.bigdata.service.EventResource;
-//BTM import com.bigdata.service.ILoadBalancerService;
import com.bigdata.service.MetadataService;
import com.bigdata.service.ndx.ClientIndexView;
@@ -513,8 +512,7 @@
* critically overloaded, but that should probably be handled by
* different logic.]
*/
-//BTM ILoadBalancerService loadBalancerService = null;
-LoadBalancer loadBalancerService = null;
+ LoadBalancer loadBalancerService = null;
if (vmd.getPercentOfSplit() < resourceManager.maximumMovePercentOfSplit
&& resourceManager.maximumMovesPerTarget != 0
&& resourceManager.getLiveJournal().getName2Addr().rangeCount() > resourceManager.minimumActiveIndexPartitions
@@ -562,19 +560,15 @@
}
/**
-BTM * Return the {@link ILoadBalancerService} if it can be discovered.
-* Return the {@link LoadBalancer} service if it can be discovered.
+ * Return the load balancer service if it can be discovered.
*
-BTM * @return the {@link ILoadBalancerService} if it can be discovered and
-* @return the {@link LoadBalancer} service if it can be discovered and
+ * @return the load balancer service if it can be discovered and
* otherwise <code>null</code>.
*/
-//BTM private ILoadBalancerService getLoadBalancerService() {
-private LoadBalancer getLoadBalancerService() {
+ private LoadBalancer getLoadBalancerService() {
// lookup the load balancer service.
-//BTM final ILoadBalancerService loadBalancerService;
-final LoadBalancer loadBalancerService;
+ final LoadBalancer loadBalancerService;
try {
@@ -612,8 +606,7 @@
* @param loadBalancerService
* The load balancer.
*/
-//BTM protected boolean shouldMove(final ILoadBalancerService loadBalancerService) {
-protected boolean shouldMove(final LoadBalancer loadBalancerService) {
+ protected boolean shouldMove(final LoadBalancer loadBalancerService) {
if (loadBalancerService == null)
throw new IllegalArgumentException();
@@ -692,8 +685,7 @@
* behavior).
*/
private UUID getMoveTarget(final UUID sourceServiceUUID,
-final LoadBalancer loadBalancerService) {
-//BTM final ILoadBalancerService loadBalancerService) {
+ final LoadBalancer loadBalancerService) {
try {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -372,8 +372,7 @@
* Counters that aggregate across all tasks submitted by the client against
* the connected federation. Those counters are sampled by a
* {@link ThreadPoolExecutorStatisticsTask} and reported by the client to
-BTM * the {@link ILoadBalancerService}.
-* the {@link LoadBalancer} service.
+ * the load balancer service.
*/
private final TaskCounters taskCounters = new TaskCounters();
@@ -401,8 +400,7 @@
* performed by the client against the connected federation. These
* {@link TaskCounters} are sampled by a
* {@link ThreadPoolExecutorStatisticsTask} and the sampled data are
-BTM * reported by the client to the {@link ILoadBalancerService}.
-* reported by the client to the {@link LoadBalancer} service.
+ * reported by the client to the load balancer service.
*/
public TaskCounters getTaskCounters() {
@@ -415,8 +413,7 @@
* for this client. There is only a single instance per scale-out index and
* all operations by this client on that index are aggregated by that
* instance. These counters are reported by the client to the
-BTM * {@link ILoadBalancerService}.
-* {@link LoadBalancer} service.
+ * load balancer service.
*
* @param name
* The scale-out index name.
@@ -456,8 +453,7 @@
/**
* Collects interesting statistics on the client's host and process
-BTM * for reporting to the {@link ILoadBalancerService}.
-* for reporting to the {@link LoadBalancer} service.
+ * for reporting to the load balancer service.
*/
private AbstractStatisticsCollector statisticsCollector;
@@ -466,15 +462,13 @@
* or until the federation is {@link #shutdown()}.
* <p>
* Note: Tasks run on this service generally update sampled values on
-BTM * {@link ICounter}s reported to the {@link ILoadBalancerService}. Basic
-* {@link ICounter}s reported to the {@link LoadBalancer} service. Basic
+ * {@link ICounter}s reported to the load balancer service. Basic
* information on the {@link #getExecutorService()} is reported
* automatically. Clients may add additional tasks to report on client-side
* aspects of their application.
* <p>
* Note: Non-sampled counters are automatically conveyed to the
-BTM * {@link ILoadBalancerService} once added to the basic {@link CounterSet}
-* {@link LoadBalancer} service once added to the basic {@link CounterSet}
+ * load balancer service once added to the basic {@link CounterSet}
* returned by {@link #getCounterSet()}.
*
* @param task
@@ -681,8 +675,7 @@
if (dataServiceUUID == null) {
-//BTM final ILoadBalancerService loadBalancerService = getLoadBalancerService();
-final LoadBalancer loadBalancerService = getLoadBalancerService();
+ final LoadBalancer loadBalancerService = getLoadBalancerService();
if (loadBalancerService == null) {
@@ -827,8 +820,7 @@
/**
* Forces the immediate reporting of the {@link CounterSet} to the
-BTM * {@link ILoadBalancerService}. Any errors will be logged, not thrown.
-* {@link LoadBalancer} service. Any errors will be logged, not thrown.
+ * load balancer service. Any errors will be logged, not thrown.
*/
public void reportCounters() {
@@ -999,10 +991,8 @@
* the (required) {@link ReportTask}.
* <p>
* Note: The {@link ReportTask} will relay any collected performance
-BTM * counters to the {@link ILoadBalancerService}, but it also lets the
-BTM * {@link ILoadBalancerService} know which services exist, which is
-* counters to the {@link LoadBalancer} service, but it also lets the
-* {@link LoadBalancer} service know which services exist, which is
+ * counters to the load balancer service, but it also lets the
+ * load balancer service know which services exist, which is
* important for some of its functions.
* <p>
*
@@ -1304,8 +1294,7 @@
/**
* Periodically report performance counter data to the
-BTM * {@link ILoadBalancerService}.
-* {@link LoadBalancer} service.
+ * load balancer service.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
@@ -1382,8 +1371,7 @@
}
-//BTM final ILoadBalancerService loadBalancerService = fed.getLoadBalancerService();
-final LoadBalancer loadBalancerService = fed.getLoadBalancerService();
+ final LoadBalancer loadBalancerService = fed.getLoadBalancerService();
if (loadBalancerService == null) {
System.out.println(">>>>> AbstractFederation.reportPerformanceCounters: loadBalancerService = NULL");
@@ -1483,8 +1471,7 @@
}
/**
-BTM * Queues up an event to be sent to the {@link ILoadBalancerService}.
-* Queues up an event to be sent to the {@link LoadBalancer} service.
+ * Queues up an event to be sent to the load balancer service.
* Events are maintained on a non-blocking queue (no fixed capacity) and
* sent by a scheduled task.
*
@@ -1503,14 +1490,12 @@
}
/**
-BTM * Queue of events sent periodically to the {@link ILoadBalancerService}.
-* Queue of events sent periodically to the {@link LoadBalancer} service.
+ * Queue of events sent periodically to the load balancer service.
*/
final private BlockingQueue<Event> events = new LinkedBlockingQueue<Event>();
/**
-BTM * Sends events to the {@link ILoadBalancerService}.
-* Sends events to the {@link LoadBalancer} service.
+ * Sends events to the load balancer service.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
@@ -1530,8 +1515,7 @@
try {
-//BTM final ILoadBalancerService lbs = getLoadBalancerService();
-final LoadBalancer lbs = getLoadBalancerService();
+ final LoadBalancer lbs = getLoadBalancerService();
if (lbs == null) {
@@ -1583,8 +1567,6 @@
} catch (Throwable t) {
log.warn(getServiceName(), t);
-//BTM
-t.printStackTrace();
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractService.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractService.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractService.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -82,7 +82,7 @@
* <p>
* Several things depend on when this method is invoked, including the setup
* of the per-service {@link CounterSet} reported by the service to the
- * {@link ILoadBalancerService}.
+ * load balancer service.
*
* @param serviceUUID
* The {@link UUID} assigned to the service.
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/DefaultServiceFederationDelegate.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/DefaultServiceFederationDelegate.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/DefaultServiceFederationDelegate.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -46,7 +46,7 @@
/**
* Basic delegate for services that need to override the service UUID and
- * service interface reported to the {@link ILoadBalancerService}.
+ * service interface reported to the load balancer service.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedClient.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedClient.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedClient.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -152,7 +152,8 @@
public static final String DATA_DIR = EmbeddedFederation.class
.getName()
+ ".dataDir";
-
+//BTM
+ public static String SERVICE_IMPL_REMOTE = "serviceImplRemote";
}
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedFederation.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedFederation.java 2010-08-06 16:03:23 UTC (rev 3425)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedFederation.java 2010-08-06 16:51:44 UTC (rev 3426)
@@ -123,11 +123,13 @@
private final ResourceLockService resourceLockManager;
/**
-BTM * The (in process) {@link LoadBalancerService}.
-* The (in process) {@link LoadBalancer} service.
+ * The (in process) load balancer service.
*/
//BTM private final LoadBalancerService loadBalancerService;
-private final EmbeddedLoadBalancer loadBalancerService;
+//private final EmbeddedLoadBalancer loadBalancerService;
+private final LoadBalancer loadBalancerService;
+private final EmbeddedLoadBalancerServiceImpl remoteLbs;
+private final EmbeddedLoadBalancerImpl lbs;
/**
* The (in process) {@link MetadataService}.
@@ -185,11 +187,9 @@
}
/**
-BTM * The (in process) {@link LoadBalancerService}.
-* The (in process) {@link LoadBalancer} service.
+ * Returns the (in process) load balancer service.
*/
-//BTM final public ILoadBalancerService getLoadBalancerService() {
-final public LoadBalancer getLoadBalancerService() {
+ final public LoadBalancer getLoadBalancerService() {
// Note: return null if service not available/discovered.
@@ -313,6 +313,8 @@
.getProperty(Options.CREATE_TEMP_FILE,
""+Options.DEFAULT_CREATE_TEMP_FILE));
+boolean serviceImplRemote = ( (null != properties.getProperty(EmbeddedClient.Options.SERVICE_IMPL_REMOTE)) ? true : false);
+System.out.println("\n*** serviceImplRemote = "+serviceImplRemote);
/*
* The directory in which the data files will reside.
*/
@@ -617,78 +619,102 @@
if (isTransient) {
-//BTM p.setProperty(LoadBalancerService.Options.TRANSIENT, "true");
-p.setProperty(EmbeddedLoadBalancerServiceImpl.Options.TRANSIENT, "true");
+ p.setProperty(LoadBalancerService.Options.TRANSIENT, "true");
+ p.setProperty(EmbeddedLoadBalancer.Options.TRANSIENT, "true");
-p.setProperty(EmbeddedLoadBalancerServiceImpl.Options.LOG_DIR,
- new File(EmbeddedLoadBalancerServiceImpl.Options.DEFAULT_LOG_DIR).toString());
-
+ p.setProperty(EmbeddedLoadBalancerImpl.Options.LOG_DIR,
+ new File
+ (EmbeddedLoadBalancerImpl.Options.DEFAULT_LOG_DIR).toString());
} else {
-
// specify the data directory for the load balancer.
-
p.setProperty(EmbeddedLoadBalancerServiceImpl.Options.LOG_DIR,
new File(dataDir, "lbs").toString());
-
+ p.setProperty(EmbeddedLoadBalancerImpl.Options.LOG_DIR,
+ new File(dataDir, "lbs").toString());
}
+ if(serviceImplRemote) {
+ try {
+ loadBalancerService =
+ new EmbeddedLoadBalancerServiceImpl
+ (UUID.randomUUID(), p).start();
+ } catch (Throwable t) {
+ log.error(t, t);
+ throw new RuntimeException(t);
+ }
+ remoteLbs = (EmbeddedLoadBalancerServiceImpl)loadBalancerService;
+ lbs = null;
+ } else {
//BTM*** remove after EmbeddedDataServiceImpl/shard.ServiceImpl/EmbeddedDataService
//BTM*** is converted to smart proxy?
-loadBalancerService = new EmbeddedLoadBalancerServiceImpl(UUID.randomUUID(),
- hostname,
- null,//SDM - replace with real SDM after conversion to smart proxy?
+ loadBalancerService =
+ new EmbeddedLoadBalancerImpl
+ (UUID.randomUUID(),
+ hostname,
+ null,//SDM - replace with real SDM after conversion to smart proxy?
//BTM*** EmbeddedDataService.this,
//BTM*** remove after EmbeddedDataService is converted to smart proxy
- dataServiceByUUID,
- p);
+ dataServiceByUUID,
+ p);
+ remoteLbs = null;
+ lbs = (EmbeddedLoadBalancerImpl)loadBalancerService;
+ }
+System.out.println("*** serviceImplRemote = "+serviceImplRemote+" >>> remoteLbs = "+remoteLbs);
+System.out.println("*** serviceImplRemote = "+serviceImplRemote+" >>> lbs = "+lbs);
//BTM*** ------------------------------------------------------------------------------
-
/*
* Have the data services join the load balancer.
*/
for (IDataService ds : this.dataService) {
-
try {
-
- loadBalancerService.join(ds.getServiceUUID(), ds
- .getServiceIface(),
-//BTM
-ds.getServiceName(),
-hostname);
-
+ if(remoteLbs != null) {
+ remoteLbs.join(ds.getServiceUUID(),
+ ds.getServiceIface(),
+ hostname);
+ } else {
+ lbs.join(ds.getServiceUUID(),
+ ds.getServiceIface(),
+ ds.getServiceName(),
+ hostname);
+ }
} catch (IOException e) {
-
// Should never be thrown for an embedded service.
-
log.warn(e.getMessage(), e);
-
}
-
}
/*
* Other service joins.
*/
+ if(remoteLbs != null) {
+ remoteLbs.join
+ (abstractTransactionService.getServiceUUID(),
+ abstractTransactionService.getServiceIface(),
+ hostname);
+ remoteLbs.join(remoteLbs.getServiceUUID(),
+ remoteLbs.getServiceIface(),
+ hostname);
+ remoteLbs.join(metadataService.getServiceUUID(),
+ metadataService.getServiceIface(),
+ hostname);
+ } else {//smart proxy
+
+ lbs.join
+ (abstractTransactionService.getServiceUUID(),
+ abstractTransactionService.getServiceIface(),
+ (abstractTransactionService.getServiceUUID()).toString(),
+ hostname);
+ lbs.join(lbs.getServiceUUID(),
+ lbs.getServiceIface(),
+ (lbs.getServiceUUID()).toString(),
+ hostname);
+ lbs.join(metadataService.getServiceUUID(),
+ metadataService.getServiceIface(),
+ (metadataService.getServiceUUID()).toString(),
+ hostname);
+ }
- loadBalancerService.join(abstractTransactionService.getServiceUUID(),
- abstractTransactionServi...
[truncated message content] |
|
From: <btm...@us...> - 2010-08-08 15:57:50
|
Revision: 3433
http://bigdata.svn.sourceforge.net/bigdata/?rev=3433&view=rev
Author: btmurphy
Date: 2010-08-08 15:57:40 +0000 (Sun, 08 Aug 2010)
Log Message:
-----------
merge -r3378:HEAD(3430) ~/bigdata/trunk ~/bigdata/branches/dev-btm [trunk --> branch dev-btm]
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/BTree.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/DumpIndexSegment.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexMetadata.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java
branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java
branches/dev-btm/bigdata/src/java/com/bigdata/io/DirectBufferPool.java
branches/dev-btm/bigdata/src/java/com/bigdata/io/WriteCache.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractJournal.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java
branches/dev-btm/bigdata/src/java/com/bigdata/mdi/PartitionLocator.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/AbstractResource.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/IMutableResource.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/RelationFusedView.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/locator/DefaultResourceLocator.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/locator/ILocatableResource.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/BTreeMetadata.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/IndexManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/OverflowManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/ResourceEvents.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/StoreManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractTransactionService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/DataService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/DefaultServiceFederationDelegate.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/DistributedTransactionService.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/GlobalRowStoreHelper.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java
branches/dev-btm/bigdata/src/java/com/bigdata/sparse/TPS.java
branches/dev-btm/bigdata/src/java/com/bigdata/striterator/ChunkedConvertingIterator.java
branches/dev-btm/bigdata/src/resources/logging/log4j.properties
branches/dev-btm/bigdata/src/test/com/bigdata/btree/AbstractIndexSegmentTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/TestIndexSegmentMultiBlockIterators.java
branches/dev-btm/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java
branches/dev-btm/bigdata/src/test/com/bigdata/concurrent/StressTestNonBlockingLockManagerWithTxDag.java
branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestAll.java
branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestTransactionService.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerBootstrapTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestReleaseResources.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/StressTestConcurrent.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestDistributedTransactionServiceRestart.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/ndx/pipeline/TestMasterTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/TransactionServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/lookup/AbstractCachingServiceClient.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/TaskMaster.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/DumpFederation.java
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query9.txt
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/DefaultExtensionFactory.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtension.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtensionFactory.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/ILexiconConfiguration.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDDecimalIV.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdWriteProc.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdWriteTask.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFDataLoadMaster.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/AbstractRuleFastClosure_3_5_6_7_9.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/RDFJoinNexus.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/LocalTripleStore.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util/Splitter.config
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util/Splitter.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/internal/ColorsEnumExtension.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/internal/EpochExtension.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/internal/SampleExtensionFactory.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestEncodeDecodeKeys.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/AbstractRIOTestCase.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/TestAsynchronousStatementBufferFactory.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/small.rdf
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/AbstractTestCase.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/TestAll.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/TestScaleOutTripleStoreWithEmbeddedFederation.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/TestScaleOutTripleStoreWithJiniFederation.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl2.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuadsAndPipelineJoins.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuadsAndPipelineJoinsWithoutInlining.java
branches/dev-btm/build.xml
branches/dev-btm/src/resources/analysis/queries/benchmark.txt
branches/dev-btm/src/resources/config/README
branches/dev-btm/src/resources/config/bigdataCluster.config
branches/dev-btm/src/resources/config/bigdataCluster16.config
branches/dev-btm/src/resources/config/log4j.properties
branches/dev-btm/src/resources/scripts/bigdata.initd
Added Paths:
-----------
branches/dev-btm/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query9-modified.txt
branches/dev-btm/src/resources/config/bigdataStandalone.config
branches/dev-btm/src/resources/scripts/dumpFed.sh
branches/dev-btm/src/resources/scripts/nanoSparqlServer.sh
Property Changed:
----------------
branches/dev-btm/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/attr/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/disco/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/
branches/dev-btm/bigdata-perf/
branches/dev-btm/bigdata-perf/lubm/lib/
branches/dev-btm/bigdata-perf/lubm/src/resources/
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util/
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/
branches/dev-btm/dsi-utils/LEGAL/
branches/dev-btm/dsi-utils/lib/
branches/dev-btm/dsi-utils/src/
branches/dev-btm/dsi-utils/src/test/
branches/dev-btm/dsi-utils/src/test/it/
branches/dev-btm/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/osgi/
branches/dev-btm/src/resources/config/
Property changes on: branches/dev-btm
___________________________________________________________________
Modified: svn:ignore
- ant-build
src
bin
bigdata*.jar
ant-release
standalone
test*
countersfinal.xml
events.jnl
.settings
*.jnl
TestInsertRate.out
SYSTAP-BBT-result.txt
U10load+query
*.hprof
com.bigdata.cache.TestHardReferenceQueueWithBatchingUpdates.exp.csv
commit-log.txt
eventLog
dist
bigdata-test
com.bigdata.rdf.stress.LoadClosureAndQueryTest.*.csv
+ ant-build
src
bin
bigdata*.jar
ant-release
standalone
test*
countersfinal.xml
events.jnl
.settings
*.jnl
TestInsertRate.out
SYSTAP-BBT-result.txt
U10load+query
*.hprof
com.bigdata.cache.TestHardReferenceQueueWithBatchingUpdates.exp.csv
commit-log.txt
eventLog
dist
bigdata-test
com.bigdata.rdf.stress.LoadClosureAndQueryTest.*.csv
DIST.*.tgz
REL.*.tgz
Modified: svn:mergeinfo
- /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/LEXICON_REFACTOR_BRANCH:2633-3304
/branches/bugfix-btm:2594-3237
/branches/fko:3150-3194
/trunk:2575-2594,2596-2877,2882-2903,2910-3378
+ /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/LEXICON_REFACTOR_BRANCH:2633-3304
/branches/bugfix-btm:2594-3237
/branches/fko:3150-3194
/trunk:2575-2594,2596-2877,2882-2903,2910-3430
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -420,7 +420,7 @@
}
}
-
+
/**
* Note: A commit is required in order for a read-committed view to have
* access to the registered indices. When running against an
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -2840,7 +2840,8 @@
* might also want to limit the maximum size of the reads.
*/
- final DirectBufferPool pool = DirectBufferPool.INSTANCE_10M;
+// final DirectBufferPool pool = DirectBufferPool.INSTANCE_10M;
+ final DirectBufferPool pool = DirectBufferPool.INSTANCE;
if (true
&& ((flags & REVERSE) == 0)
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/BTree.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/BTree.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/BTree.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -644,7 +644,18 @@
this.lastCommitTime = lastCommitTime;
}
- private long lastCommitTime = 0L;// Until the first commit.
+
+ /**
+ * The lastCommitTime of the {@link Checkpoint} record from which the
+ * {@link BTree} was loaded.
+ * <p>
+ * Note: Made volatile on 8/2/2010 since it is not otherwise obvious what
+ * would guarantee visibility of this field, through I do seem to remember
+ * that visibility might be guaranteed by how the BTree class is discovered
+ * and returned to the class. Still, it does no harm to make this a volatile
+ * read.
+ */
+ volatile private long lastCommitTime = 0L;// Until the first commit.
/**
* Return the {@link IDirtyListener}.
@@ -1525,45 +1536,63 @@
}
- /**
- * Load an instance of a {@link BTree} or derived class from the store. The
- * {@link BTree} or derived class MUST declare a constructor with the
- * following signature: <code>
+ /**
+ * Load an instance of a {@link BTree} or derived class from the store. The
+ * {@link BTree} or derived class MUST declare a constructor with the
+ * following signature: <code>
*
* <i>className</i>(IRawStore store, Checkpoint checkpoint, BTreeMetadata metadata, boolean readOnly)
*
* </code>
- *
- * @param store
- * The store.
- * @param addrCheckpoint
- * The address of a {@link Checkpoint} record for the index.
- * @param readOnly
- * When <code>true</code> the {@link BTree} will be marked as
- * read-only. Marking has some advantages relating to the locking
- * scheme used by {@link Node#getChild(int)} since the root node
- * is known to be read-only at the time that it is allocated as
- * per-child locking is therefore in place for all nodes in the
- * read-only {@link BTree}. It also results in much higher
- * concurrency for {@link AbstractBTree#touch(AbstractNode)}.
- *
- * @return The {@link BTree} or derived class loaded from that
- * {@link Checkpoint} record.
- */
+ *
+ * @param store
+ * The store.
+ * @param addrCheckpoint
+ * The address of a {@link Checkpoint} record for the index.
+ * @param readOnly
+ * When <code>true</code> the {@link BTree} will be marked as
+ * read-only. Marking has some advantages relating to the locking
+ * scheme used by {@link Node#getChild(int)} since the root node
+ * is known to be read-only at the time that it is allocated as
+ * per-child locking is therefore in place for all nodes in the
+ * read-only {@link BTree}. It also results in much higher
+ * concurrency for {@link AbstractBTree#touch(AbstractNode)}.
+ *
+ * @return The {@link BTree} or derived class loaded from that
+ * {@link Checkpoint} record.
+ *
+ * @throws IllegalArgumentException
+ * if store is <code>null</code>.
+ */
@SuppressWarnings("unchecked")
public static BTree load(final IRawStore store, final long addrCheckpoint,
final boolean readOnly) {
+ if (store == null)
+ throw new IllegalArgumentException();
+
/*
* Read checkpoint record from store.
*/
- final Checkpoint checkpoint = Checkpoint.load(store, addrCheckpoint);
+ final Checkpoint checkpoint;
+ try {
+ checkpoint = Checkpoint.load(store, addrCheckpoint);
+ } catch (Throwable t) {
+ throw new RuntimeException("Could not load Checkpoint: store="
+ + store + ", addrCheckpoint="
+ + store.toString(addrCheckpoint), t);
+ }
- /*
- * Read metadata record from store.
- */
- final IndexMetadata metadata = IndexMetadata.read(store, checkpoint
- .getMetadataAddr());
+ /*
+ * Read metadata record from store.
+ */
+ final IndexMetadata metadata;
+ try {
+ metadata = IndexMetadata.read(store, checkpoint.getMetadataAddr());
+ } catch (Throwable t) {
+ throw new RuntimeException("Could not read IndexMetadata: store="
+ + store + ", checkpoint=" + checkpoint, t);
+ }
if (log.isInfoEnabled()) {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/DumpIndexSegment.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/DumpIndexSegment.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/DumpIndexSegment.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -36,6 +36,7 @@
import org.apache.log4j.Logger;
import com.bigdata.btree.IndexSegment.ImmutableNodeFactory.ImmutableLeaf;
+import com.bigdata.io.DirectBufferPool;
import com.bigdata.journal.DumpJournal;
import com.bigdata.rawstore.IRawStore;
@@ -154,6 +155,16 @@
}
+ // multi-block scan of the index segment.
+ boolean multiBlockScan = false; // @todo command line option.
+ if (multiBlockScan) {
+
+ writeBanner("dump leaves using multi-block forward scan");
+
+ dumpLeavesMultiBlockForwardScan(store);
+
+ }
+
// dump the leaves using a fast reverse scan.
boolean fastReverseScan = true;// @todo command line option
if (fastReverseScan) {
@@ -524,6 +535,36 @@
}
+ /**
+ * Dump leaves using the {@link IndexSegmentMultiBlockIterator}.
+ *
+ * @param store
+ */
+ static void dumpLeavesMultiBlockForwardScan(final IndexSegmentStore store) {
+
+ final long begin = System.currentTimeMillis();
+
+ final IndexSegment seg = store.loadIndexSegment();
+
+ final ITupleIterator<?> itr = new IndexSegmentMultiBlockIterator(seg, DirectBufferPool.INSTANCE,
+ null/* fromKey */, null/* toKey */, IRangeQuery.DEFAULT/* flags */);
+
+ int nscanned = 0;
+
+ while(itr.hasNext()) {
+
+ itr.next();
+
+ nscanned++;
+
+ }
+
+ final long elapsed = System.currentTimeMillis() - begin;
+
+ System.out.println("Visited "+nscanned+" tuples using multi-block forward scan in "+elapsed+" ms");
+
+ }
+
static void writeBanner(String s) {
System.out.println(bar);
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexMetadata.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexMetadata.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexMetadata.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -2049,10 +2049,14 @@
// Note: default assumes NOT an index partition.
this.pmd = null;
+ /* Intern'd to reduce duplication on the heap. Will be com.bigdata.btree.BTree or
+ * com.bigdata.btree.IndexSegment and occasionally a class derived from BTree.
+ */
this.btreeClassName = getProperty(indexManager, properties, namespace,
- Options.BTREE_CLASS_NAME, BTree.class.getName().toString());
+ Options.BTREE_CLASS_NAME, BTree.class.getName()).intern();
- this.checkpointClassName = Checkpoint.class.getName();
+ // Intern'd to reduce duplication on the heap.
+ this.checkpointClassName = Checkpoint.class.getName().intern();
// this.addrSer = AddressSerializer.INSTANCE;
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -207,7 +207,7 @@
if (properties != null) {
- val = properties.getProperty(key, def);
+ val = properties.getProperty(key);//, def);
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -108,7 +108,7 @@
}
- ICUSortKeyGenerator(Locale locale, Object strength, DecompositionEnum mode) {
+ ICUSortKeyGenerator(final Locale locale, final Object strength, final DecompositionEnum mode) {
if (locale == null)
throw new IllegalArgumentException();
@@ -132,7 +132,7 @@
} else {
- StrengthEnum str = (StrengthEnum) strength;
+ final StrengthEnum str = (StrengthEnum) strength;
if (log.isInfoEnabled())
log.info("strength=" + str);
@@ -200,9 +200,9 @@
* Buffer is reused for each {@link String} from which a sort key is
* derived.
*/
- private RawCollationKey raw = new RawCollationKey(128);
+ final private RawCollationKey raw = new RawCollationKey(128);
- public void appendSortKey(KeyBuilder keyBuilder, String s) {
+ public void appendSortKey(final KeyBuilder keyBuilder, final String s) {
// RawCollationKey raw = collator.getRawCollationKey(s, null);
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -278,19 +278,19 @@
AbstractStatisticsCollector
.addGarbageCollectorMXBeanCounters(serviceRoot
.makePath(ICounterHierarchy.Memory_GarbageCollectors));
-
- /*
- * Add counters reporting on the various DirectBufferPools.
- */
- {
- // general purpose pool.
- serviceRoot.makePath(
- IProcessCounters.Memory + ICounterSet.pathSeparator
- + "DirectBufferPool").attach(
- DirectBufferPool.getCounters());
-
- }
+ // Moved since counters must be dynamically reattached to reflect pool hierarchy.
+// /*
+// * Add counters reporting on the various DirectBufferPools.
+// */
+// {
+//
+// serviceRoot.makePath(
+// IProcessCounters.Memory + ICounterSet.pathSeparator
+// + "DirectBufferPool").attach(
+// DirectBufferPool.getCounters());
+//
+// }
if (LRUNexus.INSTANCE != null) {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/io/DirectBufferPool.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/io/DirectBufferPool.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/io/DirectBufferPool.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -218,12 +218,12 @@
*/
public final static DirectBufferPool INSTANCE;
- /**
- * A JVM-wide pool of direct {@link ByteBuffer}s with a default
- * {@link Options#BUFFER_CAPACITY} of <code>10 MB</code>. The main use case
- * for the 10M buffers are multi-block IOs for the {@link IndexSegment}s.
- */
- public final static DirectBufferPool INSTANCE_10M;
+// /**
+// * A JVM-wide pool of direct {@link ByteBuffer}s with a default
+// * {@link Options#BUFFER_CAPACITY} of <code>10 MB</code>. The main use case
+// * for the 10M buffers are multi-block IOs for the {@link IndexSegment}s.
+// */
+// public final static DirectBufferPool INSTANCE_10M;
/**
* An unbounded list of all {@link DirectBufferPool} instances.
@@ -251,11 +251,11 @@
bufferCapacity//
);
- INSTANCE_10M = new DirectBufferPool(//
- "10M",//
- Integer.MAX_VALUE, // poolCapacity
- 10 * Bytes.megabyte32 // bufferCapacity
- );
+// INSTANCE_10M = new DirectBufferPool(//
+// "10M",//
+// Integer.MAX_VALUE, // poolCapacity
+// 10 * Bytes.megabyte32 // bufferCapacity
+// );
/*
* This configuration will block if there is a concurrent demand for
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/io/WriteCache.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/io/WriteCache.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/io/WriteCache.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -51,7 +51,7 @@
import com.bigdata.counters.Instrument;
import com.bigdata.journal.AbstractBufferStrategy;
import com.bigdata.journal.DiskOnlyStrategy;
-import com.bigdata.journal.DiskOnlyStrategy.StoreCounters;
+//import com.bigdata.journal.DiskOnlyStrategy.StoreCounters;
import com.bigdata.rawstore.Bytes;
import com.bigdata.rawstore.IRawStore;
import com.bigdata.rwstore.RWStore;
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractJournal.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractJournal.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractJournal.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -1027,33 +1027,33 @@
}
- case Disk: {
+// case Disk: {
+//
+// /*
+// * Setup the buffer strategy.
+// */
+//
+// fileMetadata = new FileMetadata(file, BufferMode.Disk,
+// useDirectBuffers, initialExtent, maximumExtent, create,
+// isEmptyFile, deleteOnExit, readOnly, forceWrites,
+// offsetBits, //readCacheCapacity, readCacheMaxRecordSize,
+// //readOnly ? null : writeCache,
+// writeCacheEnabled,
+// validateChecksum,
+// createTime, checker, alternateRootBlock);
+//
+// _bufferStrategy = new DiskOnlyStrategy(
+// 0L/* soft limit for maximumExtent */,
+//// minimumExtension,
+// fileMetadata);
+//
+// this._rootBlock = fileMetadata.rootBlock;
+//
+// break;
+//
+// }
- /*
- * Setup the buffer strategy.
- */
-
- fileMetadata = new FileMetadata(file, BufferMode.Disk,
- useDirectBuffers, initialExtent, maximumExtent, create,
- isEmptyFile, deleteOnExit, readOnly, forceWrites,
- offsetBits, //readCacheCapacity, readCacheMaxRecordSize,
- //readOnly ? null : writeCache,
- writeCacheEnabled,
- validateChecksum,
- createTime, checker, alternateRootBlock);
-
- _bufferStrategy = new DiskOnlyStrategy(
- 0L/* soft limit for maximumExtent */,
-// minimumExtension,
- fileMetadata);
-
- this._rootBlock = fileMetadata.rootBlock;
-
- break;
-
- }
-
-// case Disk:
+ case Disk:
case DiskWORM: {
/*
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java 2010-08-08 15:20:16 UTC (rev 3432)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java 2010-08-08 15:57:40 UTC (rev 3433)
@@ -7,6 +7,7 @@
import com.bigdata.counters.CounterSet;
import com.bigdata.counters.Instrument;
+import com.bigdata.resources.StoreManager;
import com.bigdata.service.IBigdataFederation;
import com.bigdata.service.IDataService;
@@ -171,16 +172,18 @@
* Delay between attempts reach the remote service (ms).
*/
final long delay = 10L;
-
- /**
- * #of attempts to reach the remote service.
- *
- * Note: delay*maxtries == 1000ms of trying before we give up.
- *
- * If this is not enough, then consider adding an optional parameter giving
- * the time the caller will wait and letting the StoreManager wait longer
- * during startup to discover the timestamp service.
- */
+
+ /**
+ * #of attempts to reach the remote service.
+ * <p>
+ * Note: delay*maxtries == 1000ms of trying before we give up, plus however
+ * long we are willing to wait for service discovery if the problem is
+ * locating the {@link ITransactionService}.
+ * <p>
+ * If this is not enough, then cons...
[truncated message content] |
|
From: <btm...@us...> - 2010-08-11 16:14:07
|
Revision: 3440
http://bigdata.svn.sourceforge.net/bigdata/?rev=3440&view=rev
Author: btmurphy
Date: 2010-08-11 16:14:01 +0000 (Wed, 11 Aug 2010)
Log Message:
-----------
[branch dev-btm]: fixed two minor issues that affected smart proxy deployment; added needed jar files to load balancer classpath in boot-processes.xml, and the Service interface to the uninterestingInterfaces entry in the browser.config file
Modified Paths:
--------------
branches/dev-btm/bigdata-jini/src/java/com/bigdata/boot/config/boot-processes.xml
branches/dev-btm/src/resources/bin/config/browser.config
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/boot/config/boot-processes.xml
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/boot/config/boot-processes.xml 2010-08-09 19:33:34 UTC (rev 3439)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/boot/config/boot-processes.xml 2010-08-11 16:14:01 UTC (rev 3440)
@@ -59,7 +59,7 @@
<javaprop name="java.util.logging.config.file"
value="${bigdata.configDir}/logging/logging.properties"/>
- <property name="java.classpath" value="${bootLauncherClasspath}"/>
+ <property name="java.classpath" value="${bootLauncherClasspath}${:}lib/fastutil.jar${:}lib/dsiutils.jar${:}lib/cweb-extser.jar${:}lib/icu4j.jar${:}lib/ctc_utils.jar${:}lib/lgplutils.jar"/>
<property name="java.app.mainclass"
value="com.bigdata.boot.starter.SingleNonActivatableServiceStarter"/>
<arg value="${bigdata.configDir}/policy/service.policy"/>
Modified: branches/dev-btm/src/resources/bin/config/browser.config
===================================================================
--- branches/dev-btm/src/resources/bin/config/browser.config 2010-08-09 19:33:34 UTC (rev 3439)
+++ branches/dev-btm/src/resources/bin/config/browser.config 2010-08-11 16:14:01 UTC (rev 3440)
@@ -34,6 +34,7 @@
"net.jini.admin.Administrable",
"net.jini.core.constraint.RemoteMethodControl",
"net.jini.id.ReferentUuid",
+ "com.bigdata.service.Service",
"com.bigdata.service.EventReceivingService"
};
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <sgo...@us...> - 2010-08-12 20:09:21
|
Revision: 3443
http://bigdata.svn.sourceforge.net/bigdata/?rev=3443&view=rev
Author: sgossard
Date: 2010-08-12 20:09:13 +0000 (Thu, 12 Aug 2010)
Log Message:
-----------
[dev-btm] : Updating mergeinfo for dev-btm branch to reflect correct changeset eligibility from maven_scaleout branch.
Property Changed:
----------------
branches/dev-btm/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/attr/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/disco/
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/
branches/dev-btm/bigdata-perf/
branches/dev-btm/bigdata-perf/lubm/lib/
branches/dev-btm/bigdata-perf/lubm/src/resources/
branches/dev-btm/bigdata-perf/lubm/src/resources/answers (U1)/
branches/dev-btm/bigdata-perf/lubm/src/resources/config/
branches/dev-btm/bigdata-perf/lubm/src/resources/logging/
branches/dev-btm/bigdata-perf/lubm/src/resources/scripts/
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util/
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/
branches/dev-btm/dsi-utils/LEGAL/
branches/dev-btm/dsi-utils/lib/
branches/dev-btm/dsi-utils/src/
branches/dev-btm/dsi-utils/src/test/
branches/dev-btm/dsi-utils/src/test/it/
branches/dev-btm/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom/
branches/dev-btm/osgi/
Property changes on: branches/dev-btm
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/LEXICON_REFACTOR_BRANCH:2633-3304
/branches/bugfix-btm:2594-3237
/branches/fko:3150-3194
/trunk:2575-2594,2596-2877,2882-2903,2910-3430
+ /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/LEXICON_REFACTOR_BRANCH:2633-3304
/branches/bugfix-btm:2594-3237
/branches/fko:3150-3194
/branches/maven_scaleout:3379-3439,3442
/trunk:2575-2594,2596-2877,2882-2903,2910-3430
Property changes on: branches/dev-btm/bigdata-jini/src/java/com/bigdata/attr
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/bigdata-jini/src/java/com/bigdata/attr:3270-3430
+ /branches/maven_scaleout/bigdata-jini/src/java/com/bigdata/attr:3379-3439,3442
/trunk/bigdata-jini/src/java/com/bigdata/attr:3270-3430
Property changes on: branches/dev-btm/bigdata-jini/src/java/com/bigdata/disco
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/bigdata-jini/src/java/com/bigdata/disco:3270-3430
+ /branches/maven_scaleout/bigdata-jini/src/java/com/bigdata/disco:3379-3439,3442
/trunk/bigdata-jini/src/java/com/bigdata/disco:3270-3430
Property changes on: branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/bigdata-jini/src/java/com/bigdata/util/config:3270-3430
+ /branches/maven_scaleout/bigdata-jini/src/java/com/bigdata/util/config:3379-3439,3442
/trunk/bigdata-jini/src/java/com/bigdata/util/config:3270-3430
Property changes on: branches/dev-btm/bigdata-perf
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/bigdata-perf:3270-3430
+ /branches/maven_scaleout/bigdata-perf:3379-3439,3442
/trunk/bigdata-perf:3270-3430
Property changes on: branches/dev-btm/bigdata-perf/lubm/lib
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/bigdata-perf/lubm/lib:3270-3338,3340-3430
+ /branches/maven_scaleout/bigdata-perf/lubm/lib:3379-3439,3442
/trunk/bigdata-perf/lubm/lib:3270-3338,3340-3430
Property changes on: branches/dev-btm/bigdata-perf/lubm/src/resources
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/bigdata-perf/lubm/src/resources:3270-3338,3340-3430
+ /branches/maven_scaleout/bigdata-perf/lubm/src/resources:3379-3439,3442
/trunk/bigdata-perf/lubm/src/resources:3270-3338,3340-3430
Property changes on: branches/dev-btm/bigdata-perf/lubm/src/resources/answers (U1)
___________________________________________________________________
Modified: svn:mergeinfo
-
+ /branches/maven_scaleout/bigdata-perf/lubm/src/resources/answers (U1):3379-3439,3442
Property changes on: branches/dev-btm/bigdata-perf/lubm/src/resources/config
___________________________________________________________________
Modified: svn:mergeinfo
-
+ /branches/maven_scaleout/bigdata-perf/lubm/src/resources/config:3379-3439,3442
Property changes on: branches/dev-btm/bigdata-perf/lubm/src/resources/logging
___________________________________________________________________
Modified: svn:mergeinfo
-
+ /branches/maven_scaleout/bigdata-perf/lubm/src/resources/logging:3379-3439,3442
Property changes on: branches/dev-btm/bigdata-perf/lubm/src/resources/scripts
___________________________________________________________________
Modified: svn:mergeinfo
-
+ /branches/maven_scaleout/bigdata-perf/lubm/src/resources/scripts:3379-3439,3442
Property changes on: branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/bigdata-rdf/src/java/com/bigdata/rdf/util:3270-3430
+ /branches/maven_scaleout/bigdata-rdf/src/java/com/bigdata/rdf/util:3379-3439,3442
/trunk/bigdata-rdf/src/java/com/bigdata/rdf/util:3270-3430
Property changes on: branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/bench
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/bigdata-sails/src/java/com/bigdata/rdf/sail/bench:3270-3430
+ /branches/maven_scaleout/bigdata-sails/src/java/com/bigdata/rdf/sail/bench:3379-3439,3442
/trunk/bigdata-sails/src/java/com/bigdata/rdf/sail/bench:3270-3430
Property changes on: branches/dev-btm/dsi-utils/LEGAL
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/dsi-utils/LEGAL:3270-3430
+ /branches/maven_scaleout/dsi-utils/LEGAL:3379-3439,3442
/trunk/dsi-utils/LEGAL:3270-3430
Property changes on: branches/dev-btm/dsi-utils/lib
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/dsi-utils/lib:3270-3430
+ /branches/maven_scaleout/dsi-utils/lib:3379-3439,3442
/trunk/dsi-utils/lib:3270-3430
Property changes on: branches/dev-btm/dsi-utils/src
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/dsi-utils/src:3270-3430
+ /branches/maven_scaleout/dsi-utils/src:3379-3439,3442
/trunk/dsi-utils/src:3270-3430
Property changes on: branches/dev-btm/dsi-utils/src/test
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/dsi-utils/src/test:2960-2965,2967-3430
+ /branches/maven_scaleout/dsi-utils/src/test:3379-3439,3442
/trunk/dsi-utils/src/test:2960-2965,2967-3430
Property changes on: branches/dev-btm/dsi-utils/src/test/it
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/dsi-utils/src/test/it:3270-3430
+ /branches/maven_scaleout/dsi-utils/src/test/it:3379-3439,3442
/trunk/dsi-utils/src/test/it:3270-3430
Property changes on: branches/dev-btm/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom:2960-2965,2967-3430
+ /branches/maven_scaleout/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom:3379-3439,3442
/trunk/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom:2960-2965,2967-3430
Property changes on: branches/dev-btm/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom:2960-2965,2967-3430
+ /branches/maven_scaleout/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom:3379-3439,3442
/trunk/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom:2960-2965,2967-3430
Property changes on: branches/dev-btm/osgi
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/osgi:3270-3430
+ /branches/maven_scaleout/osgi:3379-3439,3442
/trunk/osgi:3270-3430
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <btm...@us...> - 2010-08-26 22:13:44
|
Revision: 3463
http://bigdata.svn.sourceforge.net/bigdata/?rev=3463&view=rev
Author: btmurphy
Date: 2010-08-26 22:13:34 +0000 (Thu, 26 Aug 2010)
Log Message:
-----------
[branch dev-btm]: CHECKPOINT- - added smart proxy implementations of the transaction service and shard locator (metadata) service. Also made changes to allow those smart proxy implementations to be started by the ServicesManagerService or BootManager, as well as added new and changed current tests to run against either the smart proxy implementation of those services or the purely remote implemenations. Note that this work is still incomplete, and so includes numerous debug println's and BTM markers; which will be cleaned up in the final commit. Note finally that this changeset also includes hand-merged changes from the trunk for the following changesets: 3434, 3435, 3436, 3437, 3441, 3445, 3447, 3457, 3458, 3459, 3460
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/cache/RingBuffer.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/BufferMode.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/IConcurrencyManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/ILocalTransactionManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/ITransactionService.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/JournalTransactionService.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/TransactionService.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java
branches/dev-btm/bigdata/src/java/com/bigdata/mdi/PartitionLocator.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/CompactingMergeTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/ResourceManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/StoreManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractIndexCache.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractTransactionService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/CommitTimeIndex.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/DataService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/DefaultClientDelegate.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/DefaultServiceFederationDelegate.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/DistributedTransactionService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IBigdataFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IEventReceivingService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IFederationDelegate.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IMetadataService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IServiceShutdown.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IndexCache.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/MetadataIndexCache.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/NoCacheMetadataIndexView.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ShardLocator.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/AbstractScaleOutClientIndexView.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/ClientIndexView.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/RawDataServiceTupleIterator.java
branches/dev-btm/bigdata/src/resources/logging/log4j.properties
branches/dev-btm/bigdata/src/test/com/bigdata/cache/TestRingBuffer.java
branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestAll.java
branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestTransactionService.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/MockLocalTransactionManager.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/MockTransactionService.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/AbstractEmbeddedFederationTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestEventReceiver.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestMove.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestRestartSafe.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/ndx/pipeline/TestMasterTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/BigdataZooDefs.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/BigdataServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/JiniServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/MetadataServerConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServicesManagerConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/TransactionServerConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/JiniServiceProcessHelper.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/EmbeddedLoadBalancer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/AdminProxy.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/Constants.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/PrivateInterface.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/ServiceProxy.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/logging.properties
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/shardlocator.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/process/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/JiniFederation.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/LoadBalancerServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/MetadataServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/lookup/LoadBalancerClient.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/lookup/TransactionServiceClient.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/ServiceMap.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/DumpFederation.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/JiniServicesHelper.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/AdminProxy.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/Constants.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/PrivateInterface.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/ServiceProxy.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/config/logging.properties
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/Util.java
branches/dev-btm/bigdata-jini/src/resources/config/bigdataStandaloneTesting.config
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/DestroyTransactionService.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/TestServiceConfigurationMonitoring.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/TestServiceStarter.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/config/TestServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/config/testfed.config
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/testfed.config
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/master/TestMappedRDFDataLoadMaster.config
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPO.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestOptionals.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/TestLocalTripleStoreTransactionSemantics.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java
branches/dev-btm/src/resources/bin/config/browser.config
branches/dev-btm/src/resources/bin/pstart
branches/dev-btm/src/resources/config/bigdataCluster.config
Added Paths:
-----------
branches/dev-btm/bigdata/src/java/com/bigdata/service/ShardManagement.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/TxState.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/EmbeddedIndexStore.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/EmbeddedShardLocator.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/lookup/ShardLocatorClient.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/EmbeddedTransactionService.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/TestServiceConfigurationMonitoringRemote.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/TestServiceStarterRemote.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/config/TestServiceConfigurationRemote.java
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/cache/RingBuffer.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/cache/RingBuffer.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/cache/RingBuffer.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -154,7 +154,7 @@
public boolean add(final T ref) throws IllegalStateException {
if (ref == null)
- throw new IllegalArgumentException();
+ throw new NullPointerException();
beforeOffer( ref );
@@ -178,7 +178,7 @@
public boolean offer(final T ref) {
if (ref == null)
- throw new IllegalArgumentException();
+ throw new NullPointerException();
beforeOffer( ref );
@@ -387,12 +387,12 @@
if (index < 0 || index >= size)
throw new IllegalArgumentException();
- if (index + 1 == size) {
-
- // remove the LRU position.
- return remove();
-
- }
+// if (index + 1 == size) {
+//
+// // remove the LRU position.
+// return remove();
+//
+// }
/*
* Otherwise we are removing some non-LRU element.
@@ -409,7 +409,7 @@
for (;;) {
- int nexti = (i + 1) % capacity; // update index.
+ final int nexti = (i + 1) % capacity; // update index.
if (nexti != head) {
@@ -491,10 +491,9 @@
*/
final public boolean scanHead(final int nscan, final T ref) {
- assert nscan > 0;
-// if (nscan <= 0)
-// throw new IllegalArgumentException();
-//
+ if (nscan <= 0)
+ throw new IllegalArgumentException();
+
if (ref == null)
throw new IllegalArgumentException();
@@ -581,6 +580,9 @@
public boolean contains(final Object ref) {
+ if (ref == null)
+ throw new NullPointerException();
+
// MRU to LRU scan.
for (int n = 0, i = tail; n < size; n++) {
@@ -601,7 +603,8 @@
throw new NullPointerException();
if (c == this)
- throw new IllegalArgumentException();
+ return true;
+// throw new IllegalArgumentException();
for( Object e : c ) {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -178,7 +178,7 @@
* <p>
* Note: delay*maxtries == 1000ms of trying before we give up, plus however
* long we are willing to wait for service discovery if the problem is
- * locating the {@link ITransactionService}.
+ * locating the transaction service.
* <p>
* If this is not enough, then consider adding an optional parameter giving
* the time the caller will wait and letting the {@link StoreManager} wait
@@ -203,7 +203,8 @@
try {
- final ITransactionService transactionService = getTransactionService();
+//BTM final ITransactionService transactionService = getTransactionService();
+final TransactionService transactionService = getTransactionService();
if (transactionService == null) {
@@ -261,7 +262,8 @@
try {
- final ITransactionService transactionService = getTransactionService();
+//BTM final ITransactionService transactionService = getTransactionService();
+final TransactionService transactionService = getTransactionService();
if (transactionService == null) {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractTask.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractTask.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractTask.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -539,6 +539,8 @@
*/
synchronized final public ILocalBTreeView getIndex(final String name) {
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: ENTERED [name="+name+"]\n");
if (name == null) {
// @todo change to IllegalArgumentException for API consistency?
@@ -548,9 +550,13 @@
// validate that this is a declared index.
assertResource(name);
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: VALIDATED [name="+name+"]\n");
// verify still running.
assertRunning();
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: RUNNING [name="+name+"]\n");
/*
* Test the named index cache first.
@@ -560,10 +566,14 @@
final ILocalBTreeView index = indexCache.get(name);
if (index != null) {
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: INDEX NOT NULL ---> return\n");
// Cached value.
return index;
-
+}else{
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: NULL INDEX ---> continue\n");
}
}
@@ -573,6 +583,8 @@
final StaleLocatorReason reason = resourceManager.getIndexPartitionGone(name);
if (reason != null) {
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: STALE LOCATOR\n");
throw new StaleLocatorException(name, reason);
@@ -582,6 +594,8 @@
final Entry entry = n2a.get(name);
if (entry == null) {
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: NO SUCH INDEX ---> continue\n");
// index did not exist at that time.
throw new NoSuchIndexException(name);
@@ -599,11 +613,15 @@
// the unisolated name2Addr object.
final Name2Addr name2Addr = resourceManager.getLiveJournal()._getName2Addr();
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: getName2Addr\n");
synchronized (name2Addr) {
// recover from unisolated index cache.
btree = name2Addr.getIndexCache(name);
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: btree = name2Addr.getIndexCache\n");
if (btree == null) {
@@ -613,15 +631,23 @@
entry.checkpointAddr,//
false// readOnly
);
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: RE-LOADED btree from store\n");
// set the lastCommitTime on the index.
btree.setLastCommitTime(entry.commitTime);
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: SET LAST COMMIT TIME\n");
// add to the unisolated index cache (must not exist).
name2Addr.putIndexCache(name, btree, false/* replace */);
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: ADD TO INDEX CACHE\n");
if(resourceManager instanceof ResourceManager) {
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: SET BTREE COUNTERS\n");
btree
.setBTreeCounters(((ResourceManager) resourceManager)
.getIndexCounters(name));
@@ -634,9 +660,13 @@
try {
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: RETURN UNISOLATED INDEX VIEW\n");
return getUnisolatedIndexView(name, btree);
} catch (NoSuchStoreException ex) {
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: NO SUCH STORE EXCEPTION\n");
/*
* Add a little more information to the stack trace.
@@ -648,9 +678,13 @@
} else {
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: resourceManager.getIndex [name="+name+", timestamp="+timestamp+"]\n");
final ILocalBTreeView tmp = resourceManager.getIndex(name, timestamp);
if (tmp == null) {
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: tmp == null ---> NO SUCH INDEX EXCEPTION\n");
// Presume client has made a bad request
throw new NoSuchIndexException(name + ", timestamp="
@@ -664,8 +698,12 @@
* we can hold onto it for the duration of the operation.
*/
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: PUT INDEX INTO HARD REFERENCE CACHE\n");
indexCache.put(name, tmp);
+//BTM
+log.warn("\n*** AbstractTask.getIndex >>> ILocalBTreeView: EXIT ---> return cahced ILocalBTreeView\n");
return tmp;
}
@@ -1915,6 +1953,8 @@
}
public IIndex getIndex(String name) {
+//BTM
+log.warn("\n*** AbstractTask#DelegateTask.getIndex: delegate.getIndex [name="+name+"]\n");
return delegate.getIndex(name);
@@ -2215,6 +2255,8 @@
try {
+//BTM
+log.warn("\n*** AbstractTask.#IsolatedActionJournal: AbstractTask.this.getIndex [name="+name+"]\n");
return AbstractTask.this.getIndex(name);
} catch(NoSuchIndexException ex) {
@@ -2233,6 +2275,8 @@
*/
public IIndex getIndex(String name, long timestamp) {
+//BTM
+log.warn("\n*** AbstractTask#IsolatedActionJournal: getIndex [name="+name+", timestamp="+timestamp+"]\n");
if (timestamp == ITx.UNISOLATED) {
return getIndex(name);
@@ -2532,6 +2576,8 @@
* Note: Does not allow access to {@link ITx#UNISOLATED} indices.
*/
public IIndex getIndex(String name, long timestamp) {
+//BTM
+log.warn("\n*** AbstractTask#ReadOnlyJournal: getIndex [name="+name+", timestamp="+timestamp+"]\n");
if (timestamp == ITx.UNISOLATED)
throw new UnsupportedOperationException();
@@ -2562,6 +2608,8 @@
* {@link ITx#UNISOLATED} index.
*/
public IIndex getIndex(String name) {
+//BTM
+log.warn("\n*** AbstractTask#ReadOnlyJournal: getIndex [name="+name+"] >>> UnsupportedOperationException\n");
throw new UnsupportedOperationException();
@@ -2884,6 +2932,8 @@
}
public IIndex getIndex(String name, long timestamp) {
+//BTM
+log.warn("\n*** AbstractTask#DelegateIndexManager: getIndex [name="+name+", timestamp="+timestamp+"]\n");
return delegate.getIndex(name, timestamp);
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/BufferMode.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/BufferMode.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/BufferMode.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -59,15 +59,16 @@
Transient(false/* stable */, true/* fullyBuffered */),
/**
+ * <strong>This mode is not being actively developed and should not be used
+ * outside of unit tests.</strong>
* <p>
- * A direct buffer is allocated for the file image. Writes are applied
- * to the buffer. The buffer tracks dirty slots regardless of the
- * transaction that wrote them and periodically writes dirty slots
- * through to disk. On commit, any dirty index or allocation nodes are
- * written onto the buffer and all dirty slots on the buffer. Dirty
- * slots in the buffer are then synchronously written to disk, the
- * appropriate root block is updated, and the file is (optionally)
- * flushed to disk.
+ * A direct buffer is allocated for the file image. Writes are applied to
+ * the buffer. The buffer tracks dirty slots regardless of the transaction
+ * that wrote them and periodically writes dirty slots through to disk. On
+ * commit, any dirty index or allocation nodes are written onto the buffer
+ * and all dirty slots on the buffer. Dirty slots in the buffer are then
+ * synchronously written to disk, the appropriate root block is updated, and
+ * the file is (optionally) flushed to disk.
* </p>
* <p>
* This option offers wires an image of the journal file into memory and
@@ -79,6 +80,9 @@
Direct(true/* stable */, true/* fullyBuffered */),
/**
+ * <strong>This mode is not being actively developed and should not be used
+ * outside of unit tests. Memory mapped IO has the fatal weakness under Java
+ * that you can not reliably close or extend the backing file.</strong>
* <p>
* A memory-mapped buffer is allocated for the file image. Writes are
* applied to the buffer. Reads read from the buffer. On commit, the map is
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/IConcurrencyManager.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/IConcurrencyManager.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/IConcurrencyManager.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -53,12 +53,7 @@
* The client side of the transaction manager.
*/
public ILocalTransactionManager getTransactionManager();
-
-// /**
-// * The server side of the transaction manager.
-// */
-// public ITransactionManager getTransactionService();
-
+
/**
* The object used to manage local resources.
*/
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/ILocalTransactionManager.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/ILocalTransactionManager.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/ILocalTransactionManager.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -32,21 +32,21 @@
/**
* Interface for managing local transaction state (the client side of the
- * {@link ITransactionService}).
+ * transaction service).
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
*
* @todo we don't really need an interface for this as there is only one impl.
*/
-public interface ILocalTransactionManager extends
- /* ITransactionManager, */IServiceShutdown {
+public interface ILocalTransactionManager extends IServiceShutdown {
/**
* The server side of the transaction manager (possibly remote, in which
* case this may require the service to be discovered).
*/
- public ITransactionService getTransactionService();
+//BTM public ITransactionService getTransactionService();
+public TransactionService getTransactionService();
/**
* Return the local state for a transaction.
@@ -61,13 +61,13 @@
public ITx getTx(final long tx);
/**
- * Return the next timestamp from the {@link ITransactionService}.
+ * Return the next timestamp from the transaction service.
* <p>
* Note: This method is "robust" and will "retry"
- * {@link ITransactionService#nextTimestamp()} several times before giving
+ * {@link TransactionService#nextTimestamp()} several times before giving
* up.
*
- * @return The next timestamp assigned by the {@link ITransactionService}.
+ * @return The next timestamp assigned by the transaction service.
*
* @throws RuntimeException
* if the service can not be resolved or the timestamp can not
@@ -86,7 +86,7 @@
* @param commitTime
* The commit time.
*
- * @see ITransactionService#notifyCommit(long)
+ * @see TransactionService#notifyCommit(long)
*/
public void notifyCommit(long commitTime);
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/journal/ITransactionService.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/journal/ITransactionService.java 2010-08-26 16:58:31 UTC (rev 3462)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/journal/ITransactionService.java 2010-08-26 22:13:34 UTC (rev 3463)
@@ -36,6 +36,9 @@
import com.bigdata.service.IDataService;
import com.bigdata.service.ITxCommitProtocol;
+//BTM
+import com.bigdata.service.IService;
+
/**
* <p>
* An interface for managing transaction life cycles.
@@ -126,206 +129,6 @@
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
*/
-public interface ITransactionService extends ITimestampService {
+public interface ITransactionService extends TransactionService, IService {
- /**
- * Create a new transaction.
- *
- * @param timestamp
- * The timestamp may be:
- * <ul>
- * <li>A timestamp (GT ZERO), which will result in a
- * read-historical (read-only) transaction that reads from the
- * most recent committed state whose commit timestamp is less
- * than or equal to <i>timestamp</i>.</li>
- * <li>The symbolic constant {@link ITx#READ_COMMITTED} to
- * obtain a read-historical transaction reading from the most
- * recently committed state of the database. The transaction will
- * be assigned a start time corresponding to the most recent
- * commit point of the database and will be a fully isolated
- * read-only view of the state of the database as of that start
- * time. (This is an atomic shorthand for
- * newTx(getLastCommitTime())).</li>
- * <li>{@link ITx#UNISOLATED} for a read-write transaction.</li>
- * </ul>
- *
- * @return The unique transaction identifier.
- *
- * @throws IllegalStateException
- * if the requested timestamp is greater than
- * {@link #getLastCommitTime()}.
- * @throws IllegalStateException
- * if the requested timestamp is for a commit point that is no
- * longer preserved by the database (the resources for that
- * commit point have been released).
- * @throws IOException
- * RMI errors.
- *
- * @todo specialize exception for a timestamp that is no longer preserved
- * and for one that is in the future?
- */
- public long newTx(long timestamp) throws IOException;
-
- /**
- * Request commit of the transaction write set. Committing a read-only
- * transaction is necessary in order to release read locks (this is very
- * fast). If a transaction has a write set, then this method does not return
- * until that write set has been made restart safe or the transaction has
- * failed.
- * <p>
- * The commit of a transaction with a write set on a single
- * {@link IDataService} does not require either {@link ITx#UNISOLATED} tasks
- * or other transactions to wait. The latency for such commits is directly
- * related to the size of the transaction write set.
- * <p>
- * However, the commit of a transaction with writes on more than one
- * {@link IDataService} requires a distributed commit protoco...
[truncated message content] |
|
From: <btm...@us...> - 2010-10-04 18:57:20
|
Revision: 3722
http://bigdata.svn.sourceforge.net/bigdata/?rev=3722&view=rev
Author: btmurphy
Date: 2010-10-04 18:57:09 +0000 (Mon, 04 Oct 2010)
Log Message:
-----------
[branch dev-btm]: CHECKPOINT - phase 1 of the smart proxy implementation of the shard (data) service; which still includes numerous debug println's and BTM markers that will be cleaned up during phase 2. Also made appropriate additional changes to smart proxy implementations of the load balancer, the shard locator (metadata), and transaction services (from previous changesets), as required by the new smart proxy shard service. With these changes, either the ServicesManagerService mechanism or the BootManager mechanism can be used to start those service implementations. Note that this changeset also includes hand-merged changes from the trunk for the following changesets: 3481
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IRangeQuery.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexMetadata.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegment.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentStore.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/ResultSet.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/proc/AbstractKeyArrayIndexProcedureConstructor.java
branches/dev-btm/bigdata/src/java/com/bigdata/btree/proc/IIndexProcedure.java
branches/dev-btm/bigdata/src/java/com/bigdata/config/Configuration.java
branches/dev-btm/bigdata/src/java/com/bigdata/counters/CounterSet.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/ConcurrencyManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/IBTreeManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/IResourceManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/ITransactionService.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/JournalTransactionService.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/RegisterIndexTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/Tx.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java
branches/dev-btm/bigdata/src/java/com/bigdata/mdi/IMetadataIndex.java
branches/dev-btm/bigdata/src/java/com/bigdata/mdi/IPartitionMetadata.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/accesspath/AbstractAccessPath.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/accesspath/IAccessPath.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/locator/DefaultResourceLocator.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/AbstractStepTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/IJoinNexus.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/MutationTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/ProgramTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/QueryTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinMasterTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/JoinTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/JoinTaskFactoryTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/AbstractResourceManagerTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/CompactingMergeTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/IncrementalBuildTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/IndexManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/JoinIndexPartitionTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/MoveResult.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/MoveTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/OverflowManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/OverflowMetadata.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/ResourceManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/ScatterSplitTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/SplitIndexPartitionTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/SplitTailTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/StoreManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractIndexCache.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractRoundRobinServiceLoadHelper.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractScaleOutClient.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractServiceLoadHelper.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/AbstractTransactionService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/CacheOnceMetadataIndex.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/CachingMetadataIndex.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ClientService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/DataService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/DataServiceCallable.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/DistributedTransactionService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/EmbeddedFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/Event.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IBigdataClient.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IBigdataFederation.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IClientService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IDataService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IDataServiceCallable.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IRemoteExecutor.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IServiceLoadHelper.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ITxCommitProtocol.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ListIndicesTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/LoadBalancerService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/MetadataIndexCache.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/MetadataService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/NoCacheMetadataIndexView.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ShardManagement.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ShardService.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/AbstractDataServiceProcedureTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/AbstractScaleOutClientIndexView.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/ClientIndexView.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/ClientIndexViewRefactor.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/DataServiceTupleIterator.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/IClientIndex.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/IScaleOutClientIndex.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/PartitionedTupleIterator.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/RawDataServiceTupleIterator.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/SimpleDataServiceProcedureTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/pipeline/AbstractPendingSetSubtask.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/pipeline/IndexPartitionWriteTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/ndx/pipeline/IndexWriteTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/util/config/NicUtil.java
branches/dev-btm/bigdata/src/resources/logging/log4j.properties
branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestTransactionService.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/MockTransactionService.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestAddDeleteResource.java
branches/dev-btm/bigdata/src/test/com/bigdata/resources/TestBuildTask2.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/AbstractEmbeddedFederationTestCase.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/StressTestConcurrent.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestBasicIndexStuff.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestDistributedTransactionService.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestEmbeddedClient.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestEventParser.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestEventReceiver.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestMove.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestOverflow.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestRangeQuery.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestRangeQueryRemote.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestRestartSafe.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestScatterSplit.java
branches/dev-btm/bigdata/src/test/com/bigdata/service/TestSplitJoin.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/boot/config/boot-processes.xml
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/BigdataZooDefs.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ManageLogicalServiceTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServicesManagerServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/BigdataServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/DataServerConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/JiniServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/MaxDataServicesPerHostConstraint.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServicesManagerConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/EmbeddedLoadBalancer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/loadbalancer/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/Constants.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/EmbeddedShardLocator.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/PrivateInterface.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/ServiceProxy.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/logging.properties
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/ClientServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/DataServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/JiniFederation.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/MetadataServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/benchmark/ThroughputMaster.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/lookup/BigdataCachingServiceClient.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/lookup/DataServicesClient.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/AbstractAsynchronousClientTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/AbstractClientTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/AbstractResourceScanner.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/DiscoverServices.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/IAsynchronousClientTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/IResourceScannerFactory.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/MappedTaskMaster.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/ResourceBufferTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/ServiceMap.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/TaskMaster.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/DumpFederation.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/JiniServicesHelper.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/AdminProxy.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/Constants.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/PrivateInterface.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/ServiceProxy.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/config/logging.properties
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/config/shard.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/EmbeddedTransactionService.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/EntryUtil.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/Util.java
branches/dev-btm/bigdata-jini/src/resources/config/bigdataStandaloneTesting.config
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/AbstractServerTestCase.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/PerformanceTest.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/TestBigdataClient.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/master/TestMappedRDFDataLoadMaster.config
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFDataLoadMaster.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFFileLoadTask.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rules/RDFJoinNexus.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPORelation.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/store/AbstractServerTestCase.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/stress/LoadClosureAndQueryTest.java
branches/dev-btm/src/resources/bin/config/browser.config
branches/dev-btm/src/resources/config/bigdataCluster.config
Added Paths:
-----------
branches/dev-btm/bigdata/src/java/com/bigdata/counters/IDataServiceCounters.java
branches/dev-btm/bigdata/src/java/com/bigdata/counters/LoadBalancerReportingTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/counters/ReadBlockCounters.java
branches/dev-btm/bigdata/src/java/com/bigdata/counters/httpd/HttpReportingServer.java
branches/dev-btm/bigdata/src/java/com/bigdata/event/
branches/dev-btm/bigdata/src/java/com/bigdata/event/EventQueue.java
branches/dev-btm/bigdata/src/java/com/bigdata/event/EventQueueSenderTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/DistributedCommitTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/EmbeddedIndexStore.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/GetIndexMetadataTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/LocalTransactionManager.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/RangeIteratorTask.java
branches/dev-btm/bigdata/src/java/com/bigdata/journal/TxState.java
branches/dev-btm/bigdata/src/java/com/bigdata/resources/LocalResourceManagement.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/DataTaskWrapper.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/IClientServiceCallable.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/MetadataIndexCachePolicy.java
branches/dev-btm/bigdata/src/java/com/bigdata/service/OverflowAdmin.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/master/FileServer.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/EmbeddedShardService.java
Removed Paths:
-------------
branches/dev-btm/bigdata/src/java/com/bigdata/service/TxState.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/EmbeddedIndexStore.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/TestAdmin.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/TestAdmin.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/transaction/TestAdmin.java
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java 2010-10-04 13:27:07 UTC (rev 3721)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -44,7 +44,6 @@
import com.bigdata.relation.locator.DefaultResourceLocator;
import com.bigdata.search.FullTextIndex;
import com.bigdata.service.IBigdataFederation;
-import com.bigdata.service.IDataService;
import com.bigdata.sparse.AutoIncIntegerCounter;
import com.bigdata.sparse.IRowStoreConstants;
import com.bigdata.sparse.ITPS;
@@ -259,12 +258,12 @@
* <p>
* Note: The {@link BigdataFileSystem} makes the <strong>assumption</strong>
* that the {@link com.bigdata.journal.Options#OFFSET_BITS} is the #of
- * offset bits configured for the {@link IDataService}s in the connected
+ * offset bits configured for the shard services in the connected
* {@link IBigdataFederation} and computes the
* {@link BigdataFileSystem#getBlockSize()} based on that assumption. It is
* NOT possible to write blocks on the {@link BigdataFileSystem} whose size
* is greater than the maximum block size actually configured for the
- * {@link IDataService}s in the connected {@link IBigdataFederation}.
+ * shard services in the connected {@link IBigdataFederation}.
*
* @see com.bigdata.journal.Options#OFFSET_BITS
* @see #getOffsetBits()
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/IRangeQuery.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/IRangeQuery.java 2010-10-04 13:27:07 UTC (rev 3721)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/IRangeQuery.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -36,7 +36,6 @@
import com.bigdata.btree.filter.ITupleFilter;
import com.bigdata.btree.filter.TupleRemover;
import com.bigdata.service.IBigdataClient;
-import com.bigdata.service.IDataService;
import com.bigdata.service.ndx.IClientIndex;
import cutthecrap.utils.striterators.Striterator;
@@ -190,9 +189,9 @@
* <p>
* Note: This semantics of this flag require that the entries are atomically
* removed within the isolation level of the operation. In particular, if
- * the iterator is running against an {@link IDataService} using an
- * unisolated view then the entries MUST be buffered and removed as the
- * {@link ResultSet} is populated.
+ * the iterator is running against a shard service using an unisolated view
+ * then the entries MUST be buffered and removed as the {@link ResultSet}
+ * is populated.
* <p>
* Note: The {@link BigdataFileSystem#deleteHead(String, int)} relies on
* this atomic guarantee.
@@ -274,8 +273,7 @@
* processing which benefits from order effects can exploit the within chunk
* ordering of the tuples. The semantics of this flag are realized by the
* {@link IClientIndex} implementation. This flag has no effect on an
- * {@link ILocalBTreeView} and is not passed through to the
- * {@link IDataService}.
+ * {@link ILocalBTreeView} and is not passed through to the shard service.
*
* @todo This flag is not supported in combination with {@link #CURSOR}?
* @todo This flag is not supported in combination with {@link #REVERSE}?
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexMetadata.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexMetadata.java 2010-10-04 13:27:07 UTC (rev 3721)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexMetadata.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -65,12 +65,17 @@
import com.bigdata.resources.OverflowManager;
import com.bigdata.resources.StaleLocatorException;
import com.bigdata.service.AbstractFederation;
-import com.bigdata.service.DataService;
+//BTM import com.bigdata.service.DataService;
import com.bigdata.service.IBigdataFederation;
-import com.bigdata.service.IDataService;
+//BTM import com.bigdata.service.IDataService;
import com.bigdata.service.ndx.pipeline.AbstractSubtask;
import com.bigdata.sparse.SparseRowStore;
+//BTM
+import com.bigdata.service.IService;
+import com.bigdata.service.Service;
+import com.bigdata.service.ShardService;
+
/**
* <p>
* The persistent and mostly immutable metadata for a {@link AbstractBTree}.
@@ -904,7 +909,7 @@
* Boolean option indicates whether or not scatter splits are performed
* (default {@value #SCATTER_SPLIT_ENABLED}). Scatter splits only apply
* for scale-out indices where they "scatter" the initial index
- * partition across the {@link IDataService}s in the federation. This
+ * partition across the {@link ShardService}s in the federation. This
* is normally very useful.
* <P>
* Sometimes a scatter split is not the "right" thing for an index. An
@@ -1012,7 +1017,7 @@
}
/**
- * The {@link UUID} of the {@link DataService} on which the first partition
+ * The {@link UUID} of the {@link ShardService} on which the first partition
* of the scale-out index should be created. This is a purely transient
* property and will be <code>null</code> unless either explicitly set or
* set using {@value Options#INITIAL_DATA_SERVICE}. This property is only
@@ -1928,14 +1933,19 @@
final IBigdataFederation<?> fed = (IBigdataFederation<?>) indexManager;
- final IDataService dataService = fed
- .getDataServiceByName(val);
+//BTM final IDataService dataService = fed.getDataServiceByName(val);
+final ShardService dataService = fed.getDataServiceByName(val);
if (dataService != null) {
try {
- uuid = dataService.getServiceUUID();
+//BTM uuid = dataService.getServiceUUID();
+if(dataService instanceof IService) {
+ uuid = ((IService)dataService).getServiceUUID();
+} else {
+ uuid = ((Service)dataService).getServiceUUID();
+}
} catch (IOException ex) {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegment.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegment.java 2010-10-04 13:27:07 UTC (rev 3721)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegment.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -310,10 +310,15 @@
if (fileStore.fed != null) {
- openCloseEvent = new Event(fileStore.fed, new EventResource(
- fileStore.getIndexMetadata(), fileStore.file),
- EventType.IndexSegmentOpenClose);
-
+//BTM openCloseEvent = new Event(fileStore.fed, new EventResource(
+//BTM fileStore.getIndexMetadata(), fileStore.file),
+//BTM EventType.IndexSegmentOpenClose);
+openCloseEvent = new Event( (fileStore.fed).getEventQueue(),
+ (fileStore.fed).getServiceIface(),
+ (fileStore.fed).getServiceName(),
+ (fileStore.fed).getServiceUUID(),
+ new EventResource(fileStore.getIndexMetadata(), fileStore.file),
+ EventType.IndexSegmentOpenClose);
}
if (!fileStore.isOpen()) {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentStore.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentStore.java 2010-10-04 13:27:07 UTC (rev 3721)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/IndexSegmentStore.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -447,8 +447,11 @@
if (fed != null) {
- openCloseEvent = new Event(fed, new EventResource(
- indexMetadata, file),
+//BTM openCloseEvent = new Event(fed, new EventResource(
+//BTM indexMetadata, file),
+//BTM EventType.IndexSegmentStoreOpenClose).start();
+openCloseEvent = new Event( fed.getEventQueue(), fed.getServiceIface(), fed.getServiceName(), fed.getServiceUUID(),
+ new EventResource(indexMetadata, file),
EventType.IndexSegmentStoreOpenClose).start();
}
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/ResultSet.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/ResultSet.java 2010-10-04 13:27:07 UTC (rev 3721)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/ResultSet.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -55,7 +55,6 @@
import com.bigdata.io.FixedByteArrayBuffer;
import com.bigdata.journal.ITx;
import com.bigdata.mdi.IResourceMetadata;
-import com.bigdata.service.IDataService;
/**
* An object used to stream key scan results back to the client.
@@ -283,7 +282,7 @@
* The values returned by {@link ITuple#getSourceIndex()} may be used to
* identify the resource from which a given tuple was read. That information
* is used to direct {@link ITuple#readBlock(long)} requests to the correct
- * resource on the {@link IDataService}.
+ * resource on the shard service.
*/
final public IResourceMetadata[] getSources() {
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/proc/AbstractKeyArrayIndexProcedureConstructor.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/proc/AbstractKeyArrayIndexProcedureConstructor.java 2010-10-04 13:27:07 UTC (rev 3721)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/proc/AbstractKeyArrayIndexProcedureConstructor.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -5,14 +5,12 @@
import com.bigdata.btree.ITupleSerializer;
import com.bigdata.btree.IndexMetadata;
import com.bigdata.btree.raba.codec.IRabaCoder;
-import com.bigdata.service.IDataService;
/**
* A factory for {@link IKeyArrayIndexProcedure}s so that their data may be key
* range partitions and mapped against each relevant index partition.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public abstract class AbstractKeyArrayIndexProcedureConstructor<T extends IKeyArrayIndexProcedure> {
@@ -29,7 +27,8 @@
*
* @param ndx
* The index - this is used to determine the serializers for the
- * keys and/or values to be sent to a remote {@link IDataService}.
+ * keys and/or values to be sent to the backend of the shard
+ * service.
* @param fromIndex
* The index of the first key to be used (inclusive).
* @param toIndex
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/proc/IIndexProcedure.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/btree/proc/IIndexProcedure.java 2010-10-04 13:27:07 UTC (rev 3721)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/proc/IIndexProcedure.java 2010-10-04 18:57:09 UTC (rev 3722)
@@ -34,8 +34,6 @@
import com.bigdata.btree.IIndex;
import com.bigdata.btree.IRangeQuery;
import com.bigdata.btree.ISimpleBTree;
-import com.bigdata.service.DataService;
-import com.bigdata.service.IDataService;
import com.bigdata.service.ndx.ClientIndexView;
import com.bigdata.sparse.SparseRowStore;
@@ -86,15 +84,15 @@
* </dl>
*
* Note: this interface extends {@link Serializable}, however that provides
- * only for communicating state to the {@link IDataService}. If an instance of
+ * only for communicating state to the ...
[truncated message content] |
|
From: <btm...@us...> - 2010-10-04 23:12:53
|
Revision: 3728
http://bigdata.svn.sourceforge.net/bigdata/?rev=3728&view=rev
Author: btmurphy
Date: 2010-10-04 23:12:46 +0000 (Mon, 04 Oct 2010)
Log Message:
-----------
[branch dev-btm]: hand-merged changeset 3542 from trunk --> dev-btm
Modified Paths:
--------------
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFDataLoadMaster.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFFileLoadTask.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/SingleResourceReaderTask.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/BasicRioLoader.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/IRioLoader.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/PresortRioLoader.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java
branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util/Splitter.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/AbstractRIOTestCase.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/EDSAsyncLoader.java
branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/TestAsynchronousStatementBufferFactory.java
branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/stress/LoadClosureAndQueryTest.java
Modified: branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFDataLoadMaster.java
===================================================================
--- branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFDataLoadMaster.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFDataLoadMaster.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -272,6 +272,17 @@
// /** {@value #DEFAULT_MAX_TRIES} */
// int DEFAULT_MAX_TRIES = 3;
+ /**
+ * The value that will be used for the graph/context co-ordinate when
+ * loading data represented in a triple format into a quad store.
+ */
+ String DEFAULT_GRAPH = "defaultGraph" ;
+
+ /**
+ * TODO Should we always enforce a real value? i.e. provide a real default
+ * or abort the load.
+ */
+ String DEFAULT_DEFAULT_GRAPH = null ;
}
/**
@@ -400,6 +411,12 @@
private transient RDFFormat rdfFormat;
/**
+ * The value that will be used for the graph/context co-ordinate when
+ * loading data represented in a triple format into a quad store.
+ */
+ public final String defaultGraph ;
+
+ /**
* Force the load of the NxParser integration class and its registration
* of the NQuadsParser#nquads RDFFormat.
*
@@ -494,6 +511,8 @@
sb.append(", " + ConfigurationOptions.RDF_FORMAT + "=" + rdfFormat);
+ sb.append(", " + ConfigurationOptions.DEFAULT_GRAPH + "=" + defaultGraph) ;
+
sb.append(", " + ConfigurationOptions.FORCE_OVERFLOW_BEFORE_CLOSURE + "="
+ forceOverflowBeforeClosure);
@@ -624,6 +643,10 @@
}
+ defaultGraph = (String) config.getEntry(component,
+ ConfigurationOptions.DEFAULT_GRAPH, String.class,
+ ConfigurationOptions.DEFAULT_DEFAULT_GRAPH);
+
rejectedExecutionDelay = (Long) config.getEntry(
component,
ConfigurationOptions.REJECTED_EXECUTION_DELAY, Long.TYPE,
@@ -1005,6 +1028,7 @@
jobState.ontology,//file
jobState.ontology.getPath(),//baseURI
jobState.getRDFFormat(),//
+ jobState.defaultGraph,
jobState.ontologyFileFilter //
);
Modified: branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFFileLoadTask.java
===================================================================
--- branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFFileLoadTask.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFFileLoadTask.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -213,7 +213,8 @@
jobState.producerChunkSize,//
jobState.valuesInitialCapacity,//
jobState.bnodesInitialCapacity,//
- jobState.getRDFFormat(), //
+ jobState.getRDFFormat(), //
+ jobState.defaultGraph,
parserOptions,//
false, // deleteAfter is handled by the master!
jobState.parserPoolSize, //
Modified: branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/SingleResourceReaderTask.java
===================================================================
--- branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/SingleResourceReaderTask.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/load/SingleResourceReaderTask.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -23,7 +23,6 @@
* load and data verify is just the behavior of the {@link IStatementBuffer}.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public class SingleResourceReaderTask implements Runnable {
@@ -186,7 +185,7 @@
// run the parser.
// @todo reuse the same underlying parser instance?
- loader.loadRdf(reader, baseURL, rdfFormat, parserOptions);
+ loader.loadRdf(reader, baseURL, rdfFormat, null, parserOptions);
success = true;
Modified: branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java
===================================================================
--- branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -177,7 +177,6 @@
* Note: This DOES NOT support SIDS.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*
* FIXME Modify to support SIDs. We basically need to loop in the
* {@link #workflowLatch_bufferTerm2Id} workflow state until all SIDs have been
@@ -305,7 +304,6 @@
* write API for BTree and friends).
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*
* @param <S>
* The generic type of the statement objects.
@@ -356,8 +354,14 @@
* The default {@link RDFFormat}.
*/
private final RDFFormat defaultFormat;
-
+
/**
+ * The value that will be used for the graph/context co-ordinate when
+ * loading data represented in a triple format into a quad store.
+ */
+ private final String defaultGraph;
+
+ /**
* Options for the {@link RDFParser}.
*/
private final RDFParserOptions parserOptions;
@@ -1363,7 +1367,6 @@
* load and data verify is just the behavior of the {@link IStatementBuffer}.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
protected class ParserTask implements Callable<Void> {
@@ -1423,7 +1426,7 @@
try {
// run the parser.
new PresortRioLoader(buffer).loadRdf(reader, baseURL,
- rdfFormat, parserOptions);
+ rdfFormat, defaultGraph, parserOptions);
} finally {
reader.close();
}
@@ -1490,6 +1493,9 @@
* {@link BNode}s parsed from a single document.
* @param defaultFormat
* The default {@link RDFFormat} which will be assumed.
+ * @param defaultGraph
+ * The value that will be used for the graph/context co-ordinate when
+ * loading data represented in a triple format into a quad store.
* @param parserOptions
* Options for the {@link RDFParser}.
* @param deleteAfter
@@ -1529,6 +1535,7 @@
final int valuesInitialCapacity,//
final int bnodesInitialCapacity, //
final RDFFormat defaultFormat,//
+ final String defaultGraph,//
final RDFParserOptions parserOptions,//
final boolean deleteAfter,//
final int parserPoolSize,//
@@ -1566,6 +1573,8 @@
this.defaultFormat = defaultFormat;
+ this.defaultGraph = defaultGraph;
+
this.parserOptions = parserOptions;
this.deleteAfter = deleteAfter;
@@ -2250,7 +2259,6 @@
* Task deletes a resource from the local file system.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
protected class DeleteTask implements Runnable {
@@ -2712,7 +2720,6 @@
* the array is given by {@link Split#fromIndex}.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
static private class Term2IdWriteProcAsyncResultHandler
implements
@@ -2859,7 +2866,6 @@
*
* @author <a href="mailto:tho...@us...">Bryan
* Thompson</a>
- * @version $Id$
*
* @todo something similar for the SIDs
*/
@@ -2990,7 +2996,6 @@
*
* @author <a href="mailto:tho...@us...">Bryan
* Thompson</a>
- * @version $Id$
*/
static class AsyncId2TermIndexWriteTask implements Callable<Void> {
@@ -3140,7 +3145,6 @@
*
* @author <a href="mailto:tho...@us...">Bryan
* Thompson</a>
- * @version $Id$
*/
static class AsyncSPOIndexWriteTask implements Callable<Void> {
@@ -3277,7 +3281,6 @@
*
* @author <a href="mailto:tho...@us...">Bryan
* Thompson</a>
- * @version $Id$
* @param <S>
* @param <F>
*/
@@ -4083,7 +4086,6 @@
*
* @author <a href="mailto:tho...@us...">Bryan
* Thompson</a>
- * @version $Id$
*/
private class BufferOtherWritesTask implements Callable<Void> {
@@ -4153,7 +4155,6 @@
*
* @author <a href="mailto:tho...@us...">Bryan
* Thompson</a>
- * @version $Id$
*/
private class ParserThreadPoolExecutor extends ThreadPoolExecutor {
Modified: branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/BasicRioLoader.java
===================================================================
--- branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/BasicRioLoader.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/BasicRioLoader.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -41,7 +41,6 @@
* Parses data but does not load it into the indices.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public class BasicRioLoader implements IRioLoader {
@@ -74,6 +73,8 @@
private final ValueFactory valueFactory;
+ protected String defaultGraph;
+
public BasicRioLoader(final ValueFactory valueFactory) {
if (valueFactory == null)
@@ -153,18 +154,20 @@
}
final public void loadRdf(final InputStream is, final String baseURI,
- final RDFFormat rdfFormat, final RDFParserOptions options)
+ final RDFFormat rdfFormat, final String defaultGraph,
+ final RDFParserOptions options)
throws Exception {
- loadRdf2(is, baseURI, rdfFormat, options);
+ loadRdf2(is, baseURI, rdfFormat, defaultGraph, options);
}
final public void loadRdf(final Reader reader, final String baseURI,
- final RDFFormat rdfFormat, final RDFParserOptions options)
+ final RDFFormat rdfFormat, final String defaultGraph,
+ final RDFParserOptions options)
throws Exception {
- loadRdf2(reader, baseURI, rdfFormat, options);
+ loadRdf2(reader, baseURI, rdfFormat, defaultGraph, options);
}
@@ -180,7 +183,7 @@
* @throws Exception
*/
protected void loadRdf2(final Object source, final String baseURI,
- final RDFFormat rdfFormat, final RDFParserOptions options)
+ final RDFFormat rdfFormat, final String defaultGraph, final RDFParserOptions options)
throws Exception {
if (source == null)
@@ -198,6 +201,8 @@
if (log.isInfoEnabled())
log.info("format=" + rdfFormat + ", options=" + options);
+ this.defaultGraph = defaultGraph ;
+
final RDFParser parser = getParser(rdfFormat);
// apply options to the parser
@@ -212,7 +217,7 @@
// Note: reset so that rates are correct for each source loaded.
stmtsAdded = 0;
-
+
try {
before();
Modified: branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/IRioLoader.java
===================================================================
--- branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/IRioLoader.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/IRioLoader.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -37,7 +37,6 @@
* Interface for parsing RDF data using the Sesame RIO parser.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public interface IRioLoader {
@@ -72,12 +71,14 @@
* The base URL for those data.
* @param rdfFormat
* The interchange format.
+ * @param defaultGraph
+ * The default graph.
* @param options
* Options to be applied to the {@link RDFParser}.
* @throws Exception
*/
public void loadRdf(Reader reader, String baseURL, RDFFormat rdfFormat,
- RDFParserOptions options) throws Exception;
+ String defaultGraph, RDFParserOptions options) throws Exception;
/**
* Parse RDF data.
@@ -88,11 +89,13 @@
* The base URL for those data.
* @param rdfFormat
* The interchange format.
+ * @param defaultGraph
+ * The default graph.
* @param options
* Options to be applied to the {@link RDFParser}.
* @throws Exception
*/
public void loadRdf(InputStream is, String baseURI, RDFFormat rdfFormat,
- RDFParserOptions options) throws Exception;
+ String defaultGraph, RDFParserOptions options) throws Exception;
}
Modified: branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/PresortRioLoader.java
===================================================================
--- branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/PresortRioLoader.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/rio/PresortRioLoader.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -23,17 +23,19 @@
*/
package com.bigdata.rdf.rio;
+import org.openrdf.model.Resource;
import org.openrdf.model.Statement;
import org.openrdf.model.Value;
import org.openrdf.rio.RDFHandler;
import org.openrdf.rio.RDFHandlerException;
+import com.bigdata.rdf.model.BigdataURI;
+
/**
* Statement handler for the RIO RDF Parser that writes on a
* {@link StatementBuffer}.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public class PresortRioLoader extends BasicRioLoader implements RDFHandler
{
@@ -45,6 +47,12 @@
final protected IStatementBuffer<?> buffer;
/**
+ * The value that will be used for the graph/context co-ordinate when
+ * loading data represented in a triple format into a quad store.
+ */
+ private BigdataURI defaultGraphURI = null ;
+
+ /**
* Sets up parser to load RDF.
*
* @param buffer
@@ -58,7 +66,7 @@
this.buffer = buffer;
}
-
+
/**
* bulk insert the buffered data into the store.
*/
@@ -87,8 +95,11 @@
public RDFHandler newRDFHandler() {
+ defaultGraphURI = null != defaultGraph && 4 == buffer.getDatabase ().getSPOKeyArity ()
+ ? buffer.getDatabase ().getValueFactory ().createURI ( defaultGraph )
+ : null
+ ;
return this;
-
}
public void handleStatement( final Statement stmt ) {
@@ -98,9 +109,13 @@
log.debug(stmt);
}
-
+
+ Resource graph = stmt.getContext() ;
+ if ( null == graph
+ && null != defaultGraphURI ) // only true when we know we are loading a quad store
+ graph = defaultGraphURI ;
// buffer the write (handles overflow).
- buffer.add( stmt.getSubject(), stmt.getPredicate(), stmt.getObject(), stmt.getContext() );
+ buffer.add( stmt.getSubject(), stmt.getPredicate(), stmt.getObject(), graph );
stmtsAdded++;
Modified: branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java
===================================================================
--- branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -638,7 +638,7 @@
final LoadStats totals = new LoadStats();
- loadData3(totals, reader, baseURL, rdfFormat, true/*endOfBatch*/);
+ loadData3(totals, reader, baseURL, rdfFormat, null, true/*endOfBatch*/);
return totals;
@@ -666,7 +666,7 @@
final LoadStats totals = new LoadStats();
- loadData3(totals, is, baseURL, rdfFormat, true/* endOfBatch */);
+ loadData3(totals, is, baseURL, rdfFormat, null, true/* endOfBatch */);
return totals;
@@ -702,7 +702,7 @@
final LoadStats totals = new LoadStats();
- loadData3(totals, is, baseURL, rdfFormat, true/*endOfBatch*/);
+ loadData3(totals, is, baseURL, rdfFormat, null, true/*endOfBatch*/);
return totals;
@@ -773,7 +773,7 @@
final File file = new File(resource);
if(file.exists()) {
loadFiles(totals, 0/* depth */, file.toURI().toURL(),
- baseURL, rdfFormat, filter, endOfBatch);
+ baseURL, rdfFormat, null, filter, endOfBatch);
return;
}
// not on the file system, try the classpath next
@@ -807,7 +807,7 @@
try {
- loadData3(totals, reader, baseURL, rdfFormat, endOfBatch);
+ loadData3(totals, reader, baseURL, rdfFormat, null, endOfBatch);
} catch (Exception ex) {
@@ -833,6 +833,9 @@
* The format of the file (optional, when not specified the
* format is deduced for each file in turn using the
* {@link RDFFormat} static methods).
+ * @param defaultGraph
+ * The value that will be used for the graph/context co-ordinate when
+ * loading data represented in a triple format into a quad store.
* @param filter
* A filter selecting the file names that will be loaded
* (optional). When specified, the filter MUST accept directories
@@ -843,7 +846,8 @@
* @throws IOException
*/
public LoadStats loadFiles(final URL url, final String baseURI,
- final RDFFormat rdfFormat, final FilenameFilter filter)
+ final RDFFormat rdfFormat, final String defaultGraph,
+ final FilenameFilter filter)
throws IOException {
if (url == null)
@@ -851,8 +855,8 @@
final LoadStats totals = new LoadStats();
- loadFiles(totals, 0/* depth */, url, baseURI,
- rdfFormat, filter, true/* endOfBatch */);
+ loadFiles(totals, 0/* depth */, url, baseURI, rdfFormat, defaultGraph, filter, true/* endOfBatch */
+ );
return totals;
@@ -860,7 +864,8 @@
protected void loadFiles(final LoadStats totals, final int depth,
final URL url, final String baseURI, final RDFFormat rdfFormat,
- final FilenameFilter filter, final boolean endOfBatch)
+ final String defaultGraph, final FilenameFilter filter,
+ final boolean endOfBatch)
throws IOException {
// Legacy behavior - allow local files and directories for now,
@@ -886,7 +891,7 @@
final File f = files[i];
loadFiles(totals, depth + 1, f.toURI().toURL(), baseURI,
- rdfFormat, filter,
+ rdfFormat, defaultGraph, filter,
(depth == 0 && i < files.length ? false : endOfBatch));
}
@@ -930,7 +935,7 @@
final String s = baseURI != null ? baseURI : url.toURI()
.toString();
- loadData3(totals, reader, s, rdfFormat, endOfBatch);
+ loadData3(totals, reader, s, rdfFormat, defaultGraph, endOfBatch);
return;
@@ -962,7 +967,7 @@
*/
protected void loadData3(final LoadStats totals, final Object source,
final String baseURL, final RDFFormat rdfFormat,
- final boolean endOfBatch) throws IOException {
+ final String defaultGraph, final boolean endOfBatch) throws IOException {
final long begin = System.currentTimeMillis();
@@ -985,11 +990,10 @@
}
// Setup the loader.
- final PresortRioLoader loader = new PresortRioLoader(buffer);
+ final PresortRioLoader loader = new PresortRioLoader( buffer );
// @todo review: disable auto-flush - caller will handle flush of the buffer.
// loader.setFlush(false);
-
// add listener to log progress.
loader.addRioLoaderListener( new RioLoaderListener() {
@@ -1013,12 +1017,12 @@
if(source instanceof Reader) {
- loader.loadRdf((Reader) source, baseURL, rdfFormat, parserOptions);
+ loader.loadRdf((Reader) source, baseURL, rdfFormat, defaultGraph, parserOptions);
} else if (source instanceof InputStream) {
loader.loadRdf((InputStream) source, baseURL, rdfFormat,
- parserOptions);
+ defaultGraph, parserOptions);
} else
throw new AssertionError();
@@ -1368,7 +1372,7 @@
dataLoader.loadFiles(totals, 0/* depth */,
fileOrDir.toURI().toURL(), baseURI,
- rdfFormat, filter, true/* endOfBatch */
+ rdfFormat, null, filter, true/* endOfBatch */
);
}
Modified: branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util/Splitter.java
===================================================================
--- branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util/Splitter.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util/Splitter.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -99,7 +99,6 @@
* </pre>
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public class Splitter {
@@ -606,7 +605,6 @@
* Tasks parses an RDF document, writing a new file every N statements.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
protected class ParserTask implements Callable<Void> {
@@ -714,7 +712,7 @@
try {
// run the parser.
new MyLoader(buffer).loadRdf(reader, baseURL,
- defaultRDFFormat, s.parserOptions);
+ defaultRDFFormat, null, s.parserOptions);
} finally {
reader.close();
}
@@ -737,7 +735,6 @@
* {@link StatementBuffer}.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
private
static class MyLoader extends BasicRioLoader implements RDFHandler
@@ -843,7 +840,6 @@
*
* @author <a href="mailto:tho...@us...">Bryan
* Thompson</a>
- * @version $Id$
* @param <S>
*/
private class MyStatementBuffer implements
Modified: branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/AbstractRIOTestCase.java
===================================================================
--- branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/AbstractRIOTestCase.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/AbstractRIOTestCase.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -401,7 +401,7 @@
});
- loader.loadRdf((Reader) reader, baseURI, rdfFormat, options);
+ loader.loadRdf((Reader) reader, baseURI, rdfFormat, null, options);
if (log.isInfoEnabled())
log.info("Done: " + resource);
@@ -681,7 +681,7 @@
loader.loadRdf(new BufferedReader(new InputStreamReader(
new FileInputStream(resource))), baseURI, rdfFormat,
- options);
+ null, options);
if(log.isInfoEnabled())
log.info("End of reparse: nerrors=" + nerrs + ", file="
Modified: branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/EDSAsyncLoader.java
===================================================================
--- branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/EDSAsyncLoader.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/EDSAsyncLoader.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -52,7 +52,6 @@
* </pre>
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
public class EDSAsyncLoader {
@@ -161,6 +160,7 @@
valuesInitialCapacity,//
bnodesInitialCapacity,//
RDFFormat.RDFXML, // defaultFormat
+ null, // defaultGraph
parserOptions, // parserOptions
false, // deleteAfter
poolSize, // parserPoolSize,
Modified: branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/TestAsynchronousStatementBufferFactory.java
===================================================================
--- branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/TestAsynchronousStatementBufferFactory.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-rdf/src/test/com/bigdata/rdf/rio/TestAsynchronousStatementBufferFactory.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -400,6 +400,7 @@
valuesInitialCapacity,//
bnodesInitialCapacity,//
RDFFormat.RDFXML, // defaultFormat
+ null, // defaultGraph
parserOptions, //
false, // deleteAfter
parallel?5:1, // parserPoolSize,
Modified: branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/stress/LoadClosureAndQueryTest.java
===================================================================
--- branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/stress/LoadClosureAndQueryTest.java 2010-10-04 20:53:59 UTC (rev 3727)
+++ branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/stress/LoadClosureAndQueryTest.java 2010-10-04 23:12:46 UTC (rev 3728)
@@ -1204,7 +1204,7 @@
try {
dataLoader.loadFiles(dataDir.toURI().toURL(), null/* baseURI */,
- null/* rdfFormat */, filter);
+ null/* rdfFormat */, null, /* defaultGraph */filter);
} catch (IOException ex) {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <btm...@us...> - 2010-11-11 22:00:49
|
Revision: 3937
http://bigdata.svn.sourceforge.net/bigdata/?rev=3937&view=rev
Author: btmurphy
Date: 2010-11-11 22:00:41 +0000 (Thu, 11 Nov 2010)
Log Message:
-----------
[branch dev-btm]: CHECKPOINT - completed phase 1 of callable executor (client) service smart proxy work. Made changes to allow smart proxy and/or remote service implementations to be started by ServicesManagerService, service re-ordering in RestartPersistentServices because of dependency on txn service and load balancer, added DeferredInitTask to EmbeddedShardService, added the necessary jar files to classpath of shard locator and callable executor services in boot-processes.xml
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/resources/StoreManager.java
branches/dev-btm/bigdata/src/resources/logging/log4j.properties
branches/dev-btm/bigdata-jini/src/java/com/bigdata/boot/config/boot-processes.xml
branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/config/executor.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/config/logging.properties
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/RestartPersistentServices.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServiceConfigurationZNodeMonitorTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServiceConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/logging.properties
branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/shardlocator.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/EmbeddedShardService.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/config/logging.properties
branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/config/shard.config
branches/dev-btm/bigdata-jini/src/test/com/bigdata/service/jini/master/TestMappedRDFDataLoadMaster.config
branches/dev-btm/src/resources/config/bigdataCluster.config
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/resources/StoreManager.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/resources/StoreManager.java 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/resources/StoreManager.java 2010-11-11 22:00:41 UTC (rev 3937)
@@ -1523,6 +1523,10 @@
// Wait no more than N seconds for discovery
int nWait = 120;
boolean discoveredTxnSrvc = false;
+ if (log.isDebugEnabled()) {
+ log.debug("waiting for transaction "
+ +"service discovery ...");
+ }
for(int i=0; i<nWait; i++) {
if (discoveryMgr.getTransactionService() != null) {
discoveredTxnSrvc = true;
@@ -1531,24 +1535,19 @@
try {
Thread.sleep(1000L);
} catch(InterruptedException ie) { }
+ }
+ if(discoveredTxnSrvc) {
if (log.isDebugEnabled()) {
- log.debug("waiting for transaction "
- +"service discovery");
+ log.debug("StoreManager - discovered "
+ +"transaction service");
}
- if(discoveredTxnSrvc) {
- if (log.isDebugEnabled()) {
- log.debug("discovered transaction service");
- }
- } else {
- log.warn("transaction service unreachable");
- }//endif(discoveredTxnSrvc)
- }//endloop(nWait)
-//BTM if(discoveredTxnSrvc) {
-//BTM com.bigdata.util.Util.printStr("TestBigdataClientRemote.txt","\nStoreManager.start >>> TRANSACTION SERVICE DISCOVERED");
-//BTM }else{
-//BTM com.bigdata.util.Util.printStr("TestBigdataClientRemote.txt","\nStoreManager.start >>> TRANSACTION SERVICE UNREACHABLE\n");
-//BTM }
+log.warn("DISCOVERED TRANSACTION SERVICE");
+ } else {
+ log.warn("StoreManager - transaction "
+ +"service unreachable");
+ }
+
} catch (UnsupportedOperationException ex) {
//BTM com.bigdata.util.Util.printStr("TestBigdataClientRemote.txt","\nStoreManager.start >>> FEDERATION UNAVAILABLE - test case?\n");
log.warn("Federation not available - running in test case?");
Modified: branches/dev-btm/bigdata/src/resources/logging/log4j.properties
===================================================================
--- branches/dev-btm/bigdata/src/resources/logging/log4j.properties 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata/src/resources/logging/log4j.properties 2010-11-11 22:00:41 UTC (rev 3937)
@@ -215,3 +215,4 @@
log4j.logger.com.bigdata.transaction=DEBUG
log4j.logger.com.bigdata.metadata=DEBUG
log4j.logger.com.bigdata.shard=DEBUG
+log4j.logger.com.bigdata.executor=DEBUG
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/boot/config/boot-processes.xml
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/boot/config/boot-processes.xml 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/boot/config/boot-processes.xml 2010-11-11 22:00:41 UTC (rev 3937)
@@ -77,7 +77,7 @@
<javaprop name="java.util.logging.config.file"
value="${bigdata.configDir}/logging/logging.properties"/>
- <property name="java.classpath" value="${bootLauncherClasspath}${:}lib/fastutil.jar${:}lib/dsiutils.jar${:}lib/cweb-extser.jar"/>
+ <property name="java.classpath" value="${bootLauncherClasspath}${:}lib/fastutil.jar${:}lib/dsiutils.jar${:}lib/cweb-extser.jar${:}lib/ctc_utils.jar"/>
<property name="java.app.mainclass"
value="com.bigdata.boot.starter.SingleNonActivatableServiceStarter"/>
<arg value="${bigdata.configDir}/policy/service.policy"/>
@@ -113,7 +113,7 @@
<javaprop name="java.util.logging.config.file"
value="${bigdata.configDir}/logging/logging.properties"/>
- <property name="java.classpath" value="${bootLauncherClasspath}"/>
+ <property name="java.classpath" value="${bootLauncherClasspath}${:}lib/fastutil.jar${:}lib/dsiutils.jar${:}lib/cweb-extser.jar${:}lib/ctc_utils.jar"/>
<property name="java.app.mainclass"
value="com.bigdata.boot.starter.SingleNonActivatableServiceStarter"/>
<arg value="${bigdata.configDir}/policy/service.policy"/>
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/config/executor.config
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/config/executor.config 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/config/executor.config 2010-11-11 22:00:41 UTC (rev 3937)
@@ -11,6 +11,10 @@
import net.jini.core.discovery.LookupLocator;
import net.jini.discovery.LookupDiscoveryManager;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Id;
+
import com.bigdata.util.config.NicUtil;
import com.bigdata.util.config.ConfigDeployUtil;
@@ -106,3 +110,15 @@
(com.bigdata.executor.serverExporterTcpServerEndpoint,
com.bigdata.executor.serverILFactory, false, false);
}
+
+//NOTE: remove once dynamic discovery of zookeeper is added
+org.apache.zookeeper.ZooKeeper {
+
+ zroot = ConfigDeployUtil.getString("federation.name");
+
+ servers = com.bigdata.executor.serverExporterIpAddr+":2888:3888";
+
+ acl = new ACL[] {
+ new ACL(ZooDefs.Perms.ALL, new Id("world", "anyone"))
+ };
+}
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/config/logging.properties
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/config/logging.properties 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/executor/config/logging.properties 2010-11-11 22:00:41 UTC (rev 3937)
@@ -37,5 +37,4 @@
#log4j.logger.com.bigdata.executor=DEBUG
#log4j.logger.com.bigdata.executor.EmbeddedCallableExecutor=DEBUG
-#log4j.logger.com.bigdata.executor.EmbeddedClientIndexStore=DEBUG
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/RestartPersistentServices.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/RestartPersistentServices.java 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/RestartPersistentServices.java 2010-11-11 22:00:41 UTC (rev 3937)
@@ -19,6 +19,9 @@
import com.bigdata.service.jini.JiniFederation;
import com.bigdata.util.InnerCause;
+//BTM - FOR_CLIENT_SERVICE
+import java.util.ArrayList;
+
/**
* Task restarts persistent physical services that should be running on this
* host but which are not discoverable using jini (not found when we query for
@@ -154,7 +157,8 @@
+ BigdataZooDefs.CONFIG;
// these are the ServiceConfigurations.
- final List<String> serviceConfigZNodes;
+//BTM - PRE_CLIENT_SERVICE final List<String> serviceConfigZNodes;
+ List<String> serviceConfigZNodes;
try {
serviceConfigZNodes = zookeeper.getChildren(zconfig, false);
@@ -171,8 +175,109 @@
if (log.isInfoEnabled())
log.info("Considering " + serviceConfigZNodes.size()
+ " service configurations");
-System.out.println("\n*********************************************************");
-System.out.println("*** RestartPersistentServices.runOnce: Considering " + serviceConfigZNodes.size()+ " service configurations");
+System.out.println("\n*** RestartPersistentServices - BEGIN *********************************************************\n");
+for (String serviceConfigZNode : serviceConfigZNodes) {
+ System.out.println("*** RestartPersistentServices.runOnce: serviceConfigZNode = "+ serviceConfigZNode);
+}
+System.out.println("\n*** RestartPersistentServices.runOnce: RE-ORDER - DATA SERVICE LAST\n");
+//BTM - FOR_CLIENT_SERVICE - BEGIN ---------------------------------------
+ // re-order because shard (data) service waits for transaction
+ // service and load balancer service
+ List<String> tmpList = new ArrayList<String>();
+
+ //1. transaction service(s)
+ for (String serviceConfigZNode : serviceConfigZNodes) {
+ if ( (serviceConfigZNode.equals
+ ("com.bigdata.transaction.ServiceImpl")) ||
+ (serviceConfigZNode.equals
+ ("com.bigdata.service.jini.TransactionServer")) )
+ {
+ tmpList.add(serviceConfigZNode);
+ }
+ }
+ //2. load balancer service(s)
+ for (String serviceConfigZNode : serviceConfigZNodes) {
+ if ( serviceConfigZNode.equals
+ ("com.bigdata.loadbalancer.ServiceImpl") ||
+ (serviceConfigZNode.equals
+ ("com.bigdata.service.jini.LoadBalancerServer")) )
+ {
+ tmpList.add(serviceConfigZNode);
+ }
+ }
+ //3. shard locator (metadata) service(s)
+ for (String serviceConfigZNode : serviceConfigZNodes) {
+ if ( serviceConfigZNode.equals
+ ("com.bigdata.metadata.ServiceImpl") ||
+ (serviceConfigZNode.equals
+ ("com.bigdata.service.jini.MetadataServer")) )
+ {
+ tmpList.add(serviceConfigZNode);
+ }
+ }
+ //4. callable executor (client) service(s)
+ for (String serviceConfigZNode : serviceConfigZNodes) {
+ if ( serviceConfigZNode.equals
+ ("com.bigdata.executor.ServiceImpl") ||
+ (serviceConfigZNode.equals
+ ("com.bigdata.service.jini.ClientServer")) )
+ {
+ tmpList.add(serviceConfigZNode);
+ }
+ }
+ //5. shard (data) service(s)
+ for (String serviceConfigZNode : serviceConfigZNodes) {
+ if ( serviceConfigZNode.equals
+ ("com.bigdata.shard.ServiceImpl") ||
+ (serviceConfigZNode.equals
+ ("com.bigdata.service.jini.DataServer")) )
+ {
+ tmpList.add(serviceConfigZNode);
+ }
+ }
+ //6. add anything that's none of the above
+ for (String serviceConfigZNode : serviceConfigZNodes) {
+
+ if ( !(serviceConfigZNode.equals
+ ("com.bigdata.transaction.ServiceImpl")) &&
+ !(serviceConfigZNode.equals
+ ("com.bigdata.service.jini.TransactionServer")) &&
+
+ !(serviceConfigZNode.equals
+ ("com.bigdata.loadbalancer.ServiceImpl")) &&
+ !(serviceConfigZNode.equals
+ ("com.bigdata.service.jini.LoadBalancerServer")) &&
+
+ !(serviceConfigZNode.equals
+ ("com.bigdata.metadata.ServiceImpl")) &&
+ !(serviceConfigZNode.equals
+ ("com.bigdata.service.jini.MetadataServer")) &&
+
+ !(serviceConfigZNode.equals
+ ("com.bigdata.executor.ServiceImpl")) &&
+ !(serviceConfigZNode.equals
+ ("com.bigdata.service.jini.ClientServer")) &&
+
+ !(serviceConfigZNode.equals
+ ("com.bigdata.shard.ServiceImpl")) &&
+ !(serviceConfigZNode.equals
+ ("com.bigdata.service.jini.DataServer")) )
+ {
+ tmpList.add(serviceConfigZNode);
+ }
+ }
+ if (tmpList.size() == serviceConfigZNodes.size()) {
+ serviceConfigZNodes = tmpList;
+ } else {
+ log.warn("reordered list size ["+tmpList.size()
+ +"] != serviceConfigZNodes size ["
+ +serviceConfigZNodes.size()+"]");
+ }
+System.out.println("*** RestartPersistentServices.runOnce: Considering " + serviceConfigZNodes.size()+ " service configurations\n");
+for (String serviceConfigZNode : serviceConfigZNodes) {
+ System.out.println("*** RestartPersistentServices.runOnce AFTER RE-ORDER: serviceConfigZNode = "+ serviceConfigZNode);
+}
+//BTM - FOR_CLIENT_SERVICE - END -------------------------------------
for (String serviceConfigZNode : serviceConfigZNodes) {
@@ -253,7 +358,7 @@
}
}
-System.out.println("*********************************************************\n");
+System.out.println("\n*** RestartPersistentServices - END *********************************************************\n");
// Success.
return true;
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServiceConfigurationZNodeMonitorTask.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServiceConfigurationZNodeMonitorTask.java 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServiceConfigurationZNodeMonitorTask.java 2010-11-11 22:00:41 UTC (rev 3937)
@@ -323,7 +323,7 @@
if (config.serviceCount != children.size()) {
// adjust the #of logical service instances (blocks).
-System.out.println("GGGG ServiceConfigurationZNodeMonitorTask.runWithLock: *** NEW LOGICAL SERVICE TASK");
+System.out.println("GGGG ServiceConfigurationZNodeMonitorTask.runWithLock: *** NEW LOGICAL SERVICE TASK >>> "+serviceConfigZPath);
config.newLogicalServiceTask(fed, listener, serviceConfigZPath,
children).call();
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServiceConfiguration.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServiceConfiguration.java 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServiceConfiguration.java 2010-11-11 22:00:41 UTC (rev 3937)
@@ -494,7 +494,6 @@
* if the service detectably did not start.
*/
public V call() throws Exception {
-System.out.println("*** ServiceConfiguration: call() [className="+className+"]");
if (log.isInfoEnabled())
log.info("config: " + this);
@@ -532,7 +531,9 @@
TimeUnit.MILLISECONDS);
// attempt to detect a service start failure.
+System.out.println("\n*** ServiceConfiguration: call() [className="+className+"] >>> awaitServiceStart [timeout="+timeout+" ms] - BEGIN\n");
awaitServiceStart(processHelper, timeout, TimeUnit.MILLISECONDS);
+System.out.println("\n*** ServiceConfiguration: call() [className="+className+"] >>> awaitServiceStart [timeout="+timeout+" ms] - END\n");
} catch (InterruptedException ex) {
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/logging.properties
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/logging.properties 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/logging.properties 2010-11-11 22:00:41 UTC (rev 3937)
@@ -37,5 +37,4 @@
#log4j.logger.com.bigdata.metadata=DEBUG
#log4j.logger.com.bigdata.metadata.EmbeddedShardLocator=DEBUG
-#log4j.logger.com.bigdata.journal.EmbeddedIndexStore=DEBUG
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/shardlocator.config
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/shardlocator.config 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/metadata/config/shardlocator.config 2010-11-11 22:00:41 UTC (rev 3937)
@@ -11,6 +11,10 @@
import net.jini.core.discovery.LookupLocator;
import net.jini.discovery.LookupDiscoveryManager;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Id;
+
import com.bigdata.util.config.NicUtil;
import com.bigdata.util.config.ConfigDeployUtil;
@@ -89,3 +93,15 @@
(com.bigdata.metadata.serverExporterTcpServerEndpoint,
com.bigdata.metadata.serverILFactory, false, false);
}
+
+//NOTE: remove once dynamic discovery of zookeeper is added
+org.apache.zookeeper.ZooKeeper {
+
+ zroot = ConfigDeployUtil.getString("federation.name");
+
+ servers = com.bigdata.metadata.serverExporterIpAddr+":2888:3888";
+
+ acl = new ACL[] {
+ new ACL(ZooDefs.Perms.ALL, new Id("world", "anyone"))
+ };
+}
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/EmbeddedShardService.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/EmbeddedShardService.java 2010-11-11 15:02:03 UTC (rev 3936)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/shard/EmbeddedShardService.java 2010-11-11 22:00:41 UTC (rev 3937)
@@ -156,12 +156,14 @@
//BTM - BEGIN - fields from AbstractFederation -------------------------------
private ScheduledFuture<?> eventTaskFuture;
private ScheduledFuture<?> lbsReportingTaskFuture;
+ private long lbsReportingPeriod;
private AbstractHTTPD httpServer;
private String httpServerUrl;//URL used to access the httpServer
private int httpdPort;
//BTM - END - fields from AbstractFederation -------------------------------
-String dbgFlnm="EmbeddedShardService.out";
+ private ScheduledFuture deferredInitTaskFuture;
+ private boolean deferredInitDone = false;
protected EmbeddedShardService
(final UUID serviceUUID,
@@ -236,116 +238,45 @@
maxStaleLocatorRetries,
logger,
this.properties);
- String httpServerPath =
- (this.localResources).getServiceCounterPathPrefix();
- try {
- this.httpServerUrl =
- "http://"
- +AbstractStatisticsCollector.fullyQualifiedHostName
- +":"+this.httpdPort+"/?path="
- +URLEncoder.encode(httpServerPath, "UTF-8");
- } catch(java.io.UnsupportedEncodingException e) {
- logger.warn("failed to initialize httpServerUrl", e);
- }
-System.out.println("\nEmbeddedShardService >>> NEW StoreManager - BEGIN");
- this.resourceMgr =
- new ShardResourceManager(this,
- this.discoveryMgr,
- this.localResources,
- this.indexMgr,
- this.properties);
-System.out.println("\nEmbeddedShardService >>> NEW StoreManager - END");
+ this.lbsReportingPeriod = lbsReportingPeriod;
- this.localTransactionMgr = new LocalTransactionManager(discoveryMgr);
- this.concurrencyMgr =
- new ConcurrencyManager(this.properties,
- this.localTransactionMgr,
- this.resourceMgr);
- //WARN: circular refs
- (this.resourceMgr).setConcurrencyManager(this.concurrencyMgr);
- (this.indexMgr).setConcurrencyManager(this.concurrencyMgr);
+ // Note that this service employs a StoreManager (in the
+ // ResourceManager) whose creation depends on the existence
+ // of a transaction service. Additionally, this service
+ // also employs counters dependent on the existence of
+ // a load balancer to which the counters send events.
+ // Unfortunately, when the ServicesManagerService is used
+ // to start this service, these dependencies can cause
+ // problems for the ServicesManagerService. This is because
+ // the order the services are started by the ServicesManagerService
+ // can be random, and if this service is the first service
+ // the ServicesManagerService attempts to start (or whose
+ // starting is attempted before the transaction service
+ // and/or the load balancer), then unless this service
+ // returns an indication to the ServicesManagerService that
+ // it has successfully started within the time period
+ // expected, the ServicesManagerService will declare that
+ // this service is faulty and kill the process in which
+ // this service was started. To address this issue, this
+ // service executes an instance of DeferredInitTask to
+ // create the ResourceManager and set up the counters and
+ // events asynchronously; which allows the transaction
+ // service and load balancer to be started and discovered
+ // after this service has been started by the
+ // ServicesManagerService.
-//BTM - from AbstractFederation constructor and addScheduledTask
-
- //start event queue/sender task (sends events every 2 secs)
-
- long sendEventsDelay = 100L;//one-time initial delay
- long sendEventsPeriod = 2000L;
- this.eventTaskFuture =
- (localResources.getScheduledExecutor()).scheduleWithFixedDelay
- (localResources.getEventQueueSender(),
- sendEventsDelay,
- sendEventsPeriod,
- TimeUnit.MILLISECONDS);
-
-//BTM - from AbstractFederation - start deferred tasks
-
- //start task to report counters to the load balancer
-
- LoadBalancerReportingTask lbsReportingTask =
- new LoadBalancerReportingTask(this.resourceMgr,
- this.concurrencyMgr,
- this.localResources,
- this.discoveryMgr,
- logger);
- this.lbsReportingTaskFuture =
- ((this.localResources).getScheduledExecutor())
+ deferredInitDone = deferredInit();
+ if (!deferredInitDone) {
+ DeferredInitTask deferredInitTask = new DeferredInitTask();
+ this.deferredInitTaskFuture =
+ ((this.localResources).getScheduledExecutor())
.scheduleWithFixedDelay
- (lbsReportingTask,
- lbsReportingPeriod,//initial delay
- lbsReportingPeriod,
+ (deferredInitTask,
+ 20L*1000L,//initial delay
+ 30L*1000L,//period
TimeUnit.MILLISECONDS);
-
- //start an http daemon from which interested parties can query
- //counter and/or statistics information with http get commands
-
- try {
- httpServer =
- new HttpReportingServer(this.httpdPort,
- this.resourceMgr,
- this.concurrencyMgr,
- this.localResources,
- logger);
- } catch (IOException e) {
- logger.error("failed to start http server "
- +"[port="+this.httpdPort
- +", path="+httpServerPath+"]", e);
- return;
}
- if(httpServer != null) {
- if( logger.isDebugEnabled() ) {
- logger.debug("started http daemon "
- +"[access URL="+this.httpServerUrl+"]");
- }
- // add counter reporting the access url to load balancer
- ((this.localResources).getServiceCounterSet())
- .addCounter
- (IServiceCounters.LOCAL_HTTPD,
- new OneShotInstrument<String>(this.httpServerUrl));
- }
-
-//BTM - BEGIN ScaleOutIndexManager
-//BTM - The call to embeddedIndexStore.didStart was previously
-//BTM - commented out during the data service conversion. But
-//BTM - the method didStart() on the original EmbeddedIndexStore and
-//BTM - AbstractFederation calls the private method setupCounters;
-//BTM - which seems to be important for at least the shard (data)
-//BTM - service. The tests still passed without calling that method,
-//BTM - but we should consider calling it at this point (the problem
-//BTM - is that it waits on the resource manager to finish
-//BTM - initializing, which waits on the transaction service to be
-//BTM - discovered). Setting up these counters seem to be important
-//BTM - only for the shard (data) service rather than the other
-//BTM - services. So we should consider adding setupCounters to this
-//BTM - class, and calling it here instead of calling
-//BTM - embeddedIndexStore.didStart() or AbstractFederation.didStart().
-//BTM -
-//BTM embeddedIndexStore.didStart();
-
- setupCounters();
-
-//BTM - END ScaleOutIndexManager
}
// Required by Service interface
@@ -394,6 +325,11 @@
logger.warn("SSSS SHARD SERVICE EmbeddedShardService.shutdown");
if (!isOpen()) return;
+ //false ==> allow in-progress tasks to complete
+ if (deferredInitTaskFuture != null) {
+ deferredInitTaskFuture.cancel(false);
+ }
+
if (concurrencyMgr != null) {
concurrencyMgr.shutdown();
}
@@ -404,9 +340,12 @@
resourceMgr.shutdown();
}
- //false ==> allow in-progress tasks to complete
- lbsReportingTaskFuture.cancel(false);
- eventTaskFuture.cancel(false);
+ if (lbsReportingTaskFuture != null) {
+ lbsReportingTaskFuture.cancel(false);
+ }
+ if (eventTaskFuture != null) {
+ eventTaskFuture.cancel(false);
+ }
if (indexMgr != null) indexMgr.destroy();
if (localResources != null) {
@@ -1026,6 +965,119 @@
return new File(resourceMgr.getDataDir(), "httpd.url");
}
+ private boolean deferredInit() {
+
+ // StoreManager depends on the transaction service
+ if (discoveryMgr.getTransactionService() == null) return false;
+
+ if (this.resourceMgr == null) {
+System.out.println("\nEmbeddedShardService >>> NEW StoreManager - BEGIN");
+ this.resourceMgr =
+ new ShardResourceManager(this,
+ this.discoveryMgr,
+ this.localResources,
+ this.indexMgr,
+ this.properties);
+System.out.println("\nEmbeddedShardService >>> NEW StoreManager - END");
+
+ this.localTransactionMgr =
+ new LocalTransactionManager(discoveryMgr);
+ this.concurrencyMgr =
+ new ConcurrencyManager(this.properties,
+ this.localTransactionMgr,
+ this.resourceMgr);
+ //WARN: circular refs
+ (this.resourceMgr).setConcurrencyManager(this.concurrencyMgr);
+ (this.indexMgr).setConcurrencyManager(this.concurrencyMgr);
+ }
+
+...
[truncated message content] |
|
From: <btm...@us...> - 2010-11-22 17:42:24
|
Revision: 3971
http://bigdata.svn.sourceforge.net/bigdata/?rev=3971&view=rev
Author: btmurphy
Date: 2010-11-22 17:42:16 +0000 (Mon, 22 Nov 2010)
Log Message:
-----------
[branch dev-btm]: CHECKPOINT - fixes and additions to QuorumPeerService ServiceImpl class and related config to support starting and stopping multiple instances, added new QuorumPeerServiceTest to verify the fixes and additions
Modified Paths:
--------------
branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractCounterSet.java
branches/dev-btm/bigdata/src/resources/logging/log4j.properties
branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/Constants.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/config/quorum.config
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/ConfigDeployUtil.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/default-deploy.properties
branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/deploy.properties
branches/dev-btm/build.xml
Added Paths:
-----------
branches/dev-btm/bigdata-jini/lib/apache/zookeeper-3.3.2.jar
branches/dev-btm/bigdata-jini/src/test/com/bigdata/quorum/
branches/dev-btm/bigdata-jini/src/test/com/bigdata/quorum/QuorumPeerServiceTest.java
Modified: branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractCounterSet.java
===================================================================
--- branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractCounterSet.java 2010-11-19 18:12:50 UTC (rev 3970)
+++ branches/dev-btm/bigdata/src/java/com/bigdata/counters/AbstractCounterSet.java 2010-11-22 17:42:16 UTC (rev 3971)
@@ -217,6 +217,8 @@
strBuf.append( new String(new byte[] {pathBytes[i]}) );
}
}
+
+//BTM - workaround for http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6935535
logger.warn("***** AbstractCounterSet.getPath: CONTAINS SLASH-SLASH: path CONVERTED = "+strBuf.toString());
}
int slashSlashIndex = path.indexOf("//");
Modified: branches/dev-btm/bigdata/src/resources/logging/log4j.properties
===================================================================
--- branches/dev-btm/bigdata/src/resources/logging/log4j.properties 2010-11-19 18:12:50 UTC (rev 3970)
+++ branches/dev-btm/bigdata/src/resources/logging/log4j.properties 2010-11-22 17:42:16 UTC (rev 3971)
@@ -211,8 +211,13 @@
#log4j.logger.com.bigdata.boot.launcher.ConfigReaderUnitTest=DEBUG
#log4j.logger.com.bigdata.process.ProcessConfigXmlHandlerTest=DEBUG
-log4j.logger.com.bigdata.loadbalancer=DEBUG
-log4j.logger.com.bigdata.transaction=DEBUG
-log4j.logger.com.bigdata.metadata=DEBUG
-log4j.logger.com.bigdata.shard=DEBUG
-log4j.logger.com.bigdata.executor=DEBUG
+# For the quorum server tests
+log4j.logger.com.bigdata.quorum.QuorumPeerServiceTest=INFO
+
+# For smart proxy implementations of the services
+#log4j.logger.com.bigdata.loadbalancer=DEBUG
+#log4j.logger.com.bigdata.transaction=DEBUG
+#log4j.logger.com.bigdata.metadata=DEBUG
+#log4j.logger.com.bigdata.shard=DEBUG
+#log4j.logger.com.bigdata.executor=DEBUG
+#log4j.logger.com.bigdata.quorum=DEBUG
Added: branches/dev-btm/bigdata-jini/lib/apache/zookeeper-3.3.2.jar
===================================================================
(Binary files differ)
Property changes on: branches/dev-btm/bigdata-jini/lib/apache/zookeeper-3.3.2.jar
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/Constants.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/Constants.java 2010-11-19 18:12:50 UTC (rev 3970)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/Constants.java 2010-11-22 17:42:16 UTC (rev 3971)
@@ -38,4 +38,9 @@
/* Name of this component; used in config entry retrieval and the logger.*/
String COMPONENT_NAME = ((Constants.class).getPackage()).getName();
String F_SEP = System.getProperty("file.separator");
+
+ // Time (in seconds) to wait for discovery of other peers on 1st start up
+ long LOWER_BOUND_PEER_DISCOVERY_PERIOD = 1L;
+ long UPPER_BOUND_PEER_DISCOVERY_PERIOD = Long.MAX_VALUE;
+ long DEFAULT_PEER_DISCOVERY_PERIOD = 5L*60L;
}
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/ServiceImpl.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/ServiceImpl.java 2010-11-19 18:12:50 UTC (rev 3970)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/ServiceImpl.java 2010-11-22 17:42:16 UTC (rev 3971)
@@ -31,6 +31,7 @@
import com.bigdata.attr.ServiceInfo;
import com.bigdata.service.QuorumPeerService;
import com.bigdata.service.QuorumPeerService.QuorumPeerData;
+import com.bigdata.util.EntryUtil;
import com.bigdata.util.Util;
import com.bigdata.util.config.ConfigDeployUtil;
import com.bigdata.util.config.LogUtil;
@@ -332,12 +333,23 @@
joinMgr = new JoinManager(outerProxy, serviceAttrs, serviceId, ldm,
null, config);
- if((peerState.getNQuorumPeers() > 1L) && (peerState.getPeerId() == 0L))
- {
- //discover all other peers to determine peer ids and server info
- //so the entire quorum peer ensemble can be initialized & persisted
- initQuorumPeerData(peerState, configStateInfo);
+ // If not standalone and if this is the very first time this
+ // service instance has started (that is, it has not previously
+ // persisted any state related to the other peers in the
+ // ensemble), then attempt to discover all other peers in the
+ // ensemble to determine peer ids and server info so the entire
+ // quorum peer ensemble can be initialized & persisted.
+ if((peerState.getNQuorumPeers() > 1L)&&(peerState.getPeerId() == 0L)) {
+ long waitPeriod =
+ Config.getLongEntry(config,
+ COMPONENT_NAME,
+ "peerDiscoveryPeriod",
+ DEFAULT_PEER_DISCOVERY_PERIOD,
+ LOWER_BOUND_PEER_DISCOVERY_PERIOD,
+ UPPER_BOUND_PEER_DISCOVERY_PERIOD);
+ initQuorumPeerData(peerState, configStateInfo, waitPeriod);
+
//update the peerId of the QuorumPeerAttr
QuorumPeerAttr tmplVal = new QuorumPeerAttr();
QuorumPeerAttr changeVal = new QuorumPeerAttr();
@@ -354,6 +366,18 @@
myidOut.flush();
myidOut.close();
+ // zookeeperJmxLog4j item is not part of the persisted state
+ Boolean defaultJmxLog4j =
+ ConfigDeployUtil.getBoolean("federation.zookeeperJmxLog4j");
+ boolean zookeeperJmxLog4j = (Boolean)config.getEntry
+ (COMPONENT_NAME,
+ "zookeeperJmxLog4j",
+ Boolean.class,
+ defaultJmxLog4j);
+ if (!zookeeperJmxLog4j) {
+ System.setProperty("zookeeper.jmx.log4j.disable", "true");
+ }
+
this.quorumPeerMainTaskExecutor = Executors.newFixedThreadPool(1);
this.quorumPeerMainTask = new QuorumPeerMainTask(configStateInfo);
this.quorumPeerMainTaskExecutor.execute(this.quorumPeerMainTask);
@@ -374,7 +398,8 @@
}
private boolean initQuorumPeerData(QuorumPeerState peerState,
- ConfigStateInfo configStateInfo)
+ ConfigStateInfo configStateInfo,
+ long peerDiscoveryPeriod)
throws IOException, ConfigurationException
{
int nPeersInEnsemble = peerState.getNQuorumPeers();
@@ -387,21 +412,26 @@
Class[] peerTmplTypes = new Class[] { QuorumPeerService.class };
QuorumPeerAttr peerAttr = new QuorumPeerAttr();
- //match on all ports
- peerAttr.peerPort = peerState.getPeerPort();
- peerAttr.electionPort = peerState.getElectionPort();
- peerAttr.clientPort = peerState.getClientPort();
+
+//BTM Ports can all be different (which helps when running multiple
+//BTM servers on the same node), so no need to do exact matching
+//BTM on the client, peer, and election ports. Thus, discover all
+//BTM servers in the ensemble by federation group and service type,
+//BTM and wildcard the port values.
+
+//BTM peerAttr.peerPort = peerState.getPeerPort();
+//BTM peerAttr.electionPort = peerState.getElectionPort();
+//BTM peerAttr.clientPort = peerState.getClientPort();
+
Entry[] peerTmplAttrs = new Entry[] { peerAttr };
-
ServiceTemplate peerTmpl = new ServiceTemplate(peerTmplId,
peerTmplTypes,
peerTmplAttrs);
-
- long nWait = 5L*60L*1000L;//wait 5 minutes, then give up
ServiceItem[] peerItems = null;
try {
peerItems = sdm.lookup(peerTmpl, nPeersInEnsemble,
- nPeersInEnsemble, null, nWait);
+ nPeersInEnsemble, null,
+ peerDiscoveryPeriod);
if((peerItems == null) || (peerItems.length < nPeersInEnsemble)) {
return false;
}
@@ -411,34 +441,63 @@
// Found all peers, including self. Set peerId based on serviceId:
// "smallest" serviceId is set to 1, next "smallest" set to 2, etc.
+ // unless a discovered service already has a non-zero peerId field
+ // in its QuroumPeerAttr attribute.
//
- // Use TreeSet to order the proxyId's from lowest to highest
+ // Use TreeMap to order the proxyId's from lowest to highest
// (the UUID elements provide a compareTo method for consistent
// ordering).
- Set<UUID> orderedProxyIdSet = new TreeSet<UUID>();
- for(int i=0; i<peerItems.length; i++) {
- orderedProxyIdSet.add
- (((QuorumPeerService)(peerItems[i].service)).getServiceUUID());
- }
+ //
+ // Also, while populating the ordered map, determine this service's
+ // own proxyId and peerId so they can be used later when constructing
+ // and persisting the map of QuorumPeerData.
+
UUID thisProxyId = peerState.getProxyId();
if(thisProxyId == null) {
throw new NullPointerException("initQuorumPeerData: "
+"null proxyId from peerState");
}
+ Long thisPeerId = 0L;//will replace this with non-zero below
- // Determine this service's own peerId and create an ordered map
- // that maps each service's proxyId to its corresponding peerId
- // so that the QuorumPeerData map can be constructed and persisted.
- long thisPeerId = 0L;
+ // Populate the map with the ordered proxy id keys and either
+ // a non-zero peer id (indicating the discovered service had a
+ // previously initialized - and persisted - peerId) or 0 to
+ // indicate that the discovered service has been started for
+ // the very first time
+
Map<UUID, Long> orderedPeerIdMap = new TreeMap<UUID, Long>();
- Iterator<UUID> proxyItr = orderedProxyIdSet.iterator();
- for(long peerId=1; proxyItr.hasNext(); peerId++) {
- UUID nextProxyId = proxyItr.next();
- orderedPeerIdMap.put(nextProxyId, peerId);
- if( thisProxyId.equals(nextProxyId) ) {
- thisPeerId = peerId;
+ for(int i=0; i<peerItems.length; i++) {
+ UUID proxyId =
+ ((QuorumPeerService)(peerItems[i].service)).getServiceUUID();
+ Entry[] attrs = peerItems[i].attributeSets;
+ QuorumPeerAttr quorumPeerAttr =
+ EntryUtil.getEntryByType(attrs, QuorumPeerAttr.class);
+ Long peerId = (quorumPeerAttr == null ? 0L:quorumPeerAttr.peerId);
+ orderedPeerIdMap.put(proxyId, peerId);
+ logger.debug("PUT peerId >>> ["+proxyId+", "+peerId+"]");
+ }
+
+ // Replace any 0-valued peer ids from above with a non-zero value
+ // in the correct order.
+
+ Set<Map.Entry<UUID, Long>> orderedSet = orderedPeerIdMap.entrySet();
+ Iterator<Map.Entry<UUID, Long>> itr = orderedSet.iterator();
+ for(Long peerIdCntr=1L; itr.hasNext(); peerIdCntr++ ) {
+ Map.Entry<UUID, Long> pair = itr.next();
+ UUID curProxyId = pair.getKey();
+ Long curPeerId = pair.getValue();
+ if (curPeerId == 0L) {
+ curPeerId = peerIdCntr;
+ orderedPeerIdMap.put(curProxyId, curPeerId);//replace
+ logger.debug("REPLACE peerId >>> "
+ +"["+curProxyId+", "+curPeerId+"]");
}
+ if( thisProxyId.equals(curProxyId) ) {//determine own peerId
+ thisPeerId = curPeerId;
+ }
}
+
+ //verify this service's peerId was indeed determined above
if(thisPeerId == 0) return false;
peerState.setPeerId(thisPeerId);
@@ -462,6 +521,8 @@
( new InetSocketAddress(peerAddress, peerPort) );
peerData.setElectionAddress
( new InetSocketAddress(peerAddress, electionPort) );
+
+ peerDataMap.put(peerId, peerData);
}
peerState.setPeerDataMap(peerDataMap);
@@ -718,6 +779,7 @@
(new Integer(peerState.getMaxClientCnxns())).toString());
Map<Long, QuorumPeerData> peerDataMap = peerState.getPeerDataMap();
+ logger.debug("peerDataMap.size() = "+peerDataMap.size());
for(QuorumPeerData peerData : peerDataMap.values()) {
long peerId = peerData.getPeerId();
InetSocketAddress pAddr = peerData.getPeerAddress();
@@ -728,6 +790,7 @@
String serverKey = "server."+peerId;
String serverVal = peerAddr.getHostAddress()
+":"+peerPort+":"+electionPort;
+ logger.debug("serverKey="+serverKey+", serverVal="+serverVal);
configProps.setProperty(serverKey, serverVal);
}
return configProps;
@@ -885,10 +948,10 @@
//clientPort
//for zookeeper 3.2.1
- this.peerState.setClientPort(zConfig.getClientPort());
+// this.peerState.setClientPort(zConfig.getClientPort());
//for zookeeper 3.3.0+
-// this.peerState.setClientPort
-// (zConfig.getClientPortAddress().getPort());
+ this.peerState.setClientPort
+ (zConfig.getClientPortAddress().getPort());
//dataDir
this.peerState.setDataDir(zConfig.getDataDir());
@@ -977,6 +1040,7 @@
if( peerIdFound ) break;
}
}
+
this.peerState.setPeerDataMap(peerDataMap);
this.peerState.setNQuorumPeers(peerDataMap.size());
}
@@ -984,13 +1048,17 @@
} else {//retrieve from jini config
logger.log(Level.DEBUG, "INITIAL START: "
- +"[use jini config]");
+ +"[use deployment config]");
//clientPort
+ Integer defaultClientPort =
+ ConfigDeployUtil.getInt
+ ("federation.zookeeperClientPort");
Integer zClientPort =
(Integer)config.getEntry(COMPONENT_NAME,
"zookeeperClientPort",
- Integer.class, 2181);
+ Integer.class,
+ defaultClientPort);
if(zClientPort == null) {
throw new ConfigurationException
("null zookeeperClientPort");
@@ -998,7 +1066,9 @@
this.peerState.setClientPort(zClientPort);
//dataDir
- String defaultDataDir = "data";
+ String defaultDataDir =
+ ConfigDeployUtil.getString
+ ("federation.zookeeperDataDir");
String zDataDir = persistBaseStr + F_SEP
+ (String)config.getEntry(COMPONENT_NAME,
"zookeeperDataDir",
@@ -1011,7 +1081,9 @@
this.peerState.setDataDir(zDataDir);
//dataLogDir
- String defaultDataLogDir = "data.log";
+ String defaultDataLogDir =
+ ConfigDeployUtil.getString
+ ("federation.zookeeperDataLogDir");
String zDataLogDir = persistBaseStr + F_SEP
+ (String)config.getEntry
(COMPONENT_NAME,
@@ -1025,10 +1097,14 @@
this.peerState.setDataLogDir(zDataLogDir);
//tickTime
+ Integer defaultTickTime =
+ ConfigDeployUtil.getInt
+ ("federation.zookeepeTickTime");
Integer zTickTime =
(Integer)config.getEntry(COMPONENT_NAME,
"zookeeperTickTime",
- Integer.class, 2000);
+ Integer.class,
+ defaultTickTime);
if(zTickTime == null) {
throw new ConfigurationException
("null zookeeperTickTime");
@@ -1036,6 +1112,9 @@
this.peerState.setTickTime(zTickTime);
//initLimit
+ Integer defaultInitLimit =
+ ConfigDeployUtil.getInt
+ ("federation.zookeeperInitLimit");
Integer zInitLimit =
(Integer)config.getEntry(COMPONENT_NAME,
"zookeeperInitLimit",
@@ -1047,10 +1126,14 @@
this.peerState.setInitLimit(zInitLimit);
//syncLimit
+ Integer defaultSyncLimit =
+ ConfigDeployUtil.getInt
+ ("federation.zookeeperSyncLimit");
Integer zSyncLimit =
(Integer)config.getEntry(COMPONENT_NAME,
"zookeeperSyncLimit",
- Integer.class, 2);
+ Integer.class,
+ defaultSyncLimit);
if(zSyncLimit == null) {
throw new ConfigurationException
("null zookeeperSyncLimit");
@@ -1058,10 +1141,14 @@
this.peerState.setSyncLimit(zSyncLimit);
//electionAlg
+ Integer defaultElectionAlg =
+ ConfigDeployUtil.getInt
+ ("federation.zookeeperElectionAlg");
Integer zElectionAlg =
(Integer)config.getEntry(COMPONENT_NAME,
"zookeeperElectionAlg",
- Integer.class, 3);
+ Integer.class,
+ defaultElectionAlg);
if(zElectionAlg == null) {
throw new ConfigurationException
("null zookeeperElectionAlg");
@@ -1069,10 +1156,14 @@
this.peerState.setElectionAlg(zElectionAlg);
//maxClientCnxns
+ Integer defaultMaxClientCnxns =
+ ConfigDeployUtil.getInt
+ ("federation.zookeeperMaxClientCnxns");
Integer zMaxClientCnxns =
(Integer)config.getEntry(COMPONENT_NAME,
"zookeeperMaxClientCnxns",
- Integer.class, 10);
+ Integer.class,
+ defaultMaxClientCnxns);
if(zMaxClientCnxns == null) {
throw new ConfigurationException
("null zookeeperMaxClientCnxns");
@@ -1080,10 +1171,11 @@
this.peerState.setMaxClientCnxns(zMaxClientCnxns);
// Because this is the first time this service is started,
- // and because the config is retrieved from a jini config,
- // there is no knowledge (yet) of the other zookeeper
- // servers, other than the total number of peers that
- // are expected to make up the ensemble. If the ensemble
+ // and because the config is retrieved from a deployment
+ // specific configuration, there is no knowledge (yet)
+ // of the other zookeeper servers, other than the total
+ // number of peers that are expected to make up the
+ // ensemble (zookeeperEnsembleSize). If the ensemble
// will consist of only this service peer, then the
// peerConfigMap can be created and populated with this
// service's server information. But if the ensemble will
@@ -1092,21 +1184,31 @@
// servers are discovered and a leader is elected.
//zookeeperNetwork (peerAddress)
- String zookeeperNetwork = NicUtil.getIpAddress("default.nic", ConfigDeployUtil.getString("node.serviceNetwork"), false);
+ String zookeeperNetwork =
+ NicUtil.getIpAddress
+ ("default.nic",
+ ConfigDeployUtil.getString
+ ("node.serviceNetwork"),
+ false);
if(zookeeperNetwork == null) {
throw new ConfigurationException
("null zookeeperNetwork");
}
InetAddress peerAddress =
- NicUtil.getInetAddress(zookeeperNetwork, 0, null, true);
+ NicUtil.getInetAddress
+ (zookeeperNetwork, 0, null, true);
this.peerState.setPeerAddress(peerAddress);
//peerPort
+ Integer defaultPeerPort =
+ ConfigDeployUtil.getInt
+ ("federation.zookeeperPeerPort");
Integer peerPort =
(Integer)config.getEntry(COMPONENT_NAME,
"zookeeperPeerPort",
- Integer.class, 2888);
+ Integer.class,
+ defaultPeerPort);
if(peerPort == null) {
throw new ConfigurationException
("null zookeeperPeerPort");
@@ -1114,6 +1216,9 @@
this.peerState.setPeerPort(peerPort);
//electionPort
+ Integer defaultElectionPort =
+ ConfigDeployUtil.getInt
+ ("federation.zookeeperElectionPort");
Integer electionPort =
(Integer)config.getEntry(COMPONENT_NAME,
"zookeeperElectionPort",
@@ -1125,10 +1230,14 @@
this.peerState.setElectionPort(electionPort);
//nQuorumPeers
+ Integer defaultEnsembleSize =
+ ConfigDeployUtil.getInt
+ ("federation.zookeeperEnsembleSize");
Integer nQuorumPeers =
(Integer)config.getEntry(COMPONENT_NAME,
- "nQuorumPeers",
- Integer.class, 1);
+ "zookeeperEnsembleSize",
+ Integer.class,
+ defaultEnsembleSize);
if(nQuorumPeers == null) {
throw new ConfigurationException
("null nQuorumPeers");
@@ -1140,7 +1249,18 @@
this.peerState.setNQuorumPeers(nQuorumPeers);
if(nQuorumPeers > 1) {
- this.peerState.setPeerId(0L);//0 - no peers discovered
+
+ // nQuorumPeers > 1 means that the ensemble is
+ // configured to not be standalone; in which case,
+ // this service's peer id is initially set to 0
+ // to indicate to this service's init method that
+ // the initQuorumPeerData method must be invoked
+ // to dicover the other peers so in the ensemble
+ // so that each peer's id (myid) can be set to a
+ // unique value between 1 and nQuorumPeers
+
+ this.peerState.setPeerId(0L);
+
} else {//nQuorumPeers == 1, populate peerConfigMap
long peerId = 1L;
this.peerState.setPeerId(peerId);
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/config/quorum.config
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/config/quorum.config 2010-11-19 18:12:50 UTC (rev 3970)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/config/quorum.config 2010-11-22 17:42:16 UTC (rev 3971)
@@ -58,14 +58,7 @@
( new String[] { System.getProperty("app.home", "${user.dir}"),
"${/}var${/}state${/}quorumState" } );
- zookeeperDataDir = "data";
- zookeeperDataLogDir = "data.log";
- zookeeperClientPort = new Integer(2181);
- zookeeperTickTime = new Integer(2000);
- zookeeperInitLimit = new Integer(5);
- zookeeperSyncLimit = new Integer(2);
- zookeeperElectionAlg = new Integer(3);//0 = udp, 3 = tcp
- zookeeperMaxClientCnxns = new Integer(10);
+ peerDiscoveryPeriod = 300000L;// wait 5 minutes for peer discovery
// If standard zookeeper config is specified,
// it will override jini config; for example,
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/ConfigDeployUtil.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/ConfigDeployUtil.java 2010-11-19 18:12:50 UTC (rev 3970)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/ConfigDeployUtil.java 2010-11-22 17:42:16 UTC (rev 3971)
@@ -400,10 +400,10 @@
throws ConfigurationException
{
String validValuesStr =
- (String) getDeploymentProperties().get(parameter + STRINGVALS);
+ (String) getDeploymentProperties().get(parameter + STRINGVALS);
if (validValuesStr != null) {
- String[] validValues = validValuesStr.split(",");
+ String[] validValues = (validValuesStr.trim()).split(",");
if (!Arrays.asList(validValues).contains(value)) {
throw new ConfigurationException
("invalid string parameter ["+parameter+"] in "
@@ -417,11 +417,11 @@
throws ConfigurationException
{
String validValuesStr =
- (String)(getDeploymentProperties().get(parameter + STRINGVALS));
+ (String)(getDeploymentProperties().get(parameter + STRINGVALS));
String[] values = value.split(",");
if (validValuesStr != null) {
- String[] validValues = validValuesStr.split(",");
+ String[] validValues = (validValuesStr.trim()).split(",");
List validValuesList = Arrays.asList(validValues);
for (int i=0; i<values.length; i++) {
if (!validValuesList.contains(values[i])) {
@@ -434,6 +434,39 @@
return values;
}
+//NOTE: there seems to be a problem with NumberFormat.parse; in particular,
+// with parsing strings representing integers. During testing
+// there were numerous occasions where, when parsing a string value
+// from the properties (from either default-deploy.properties or
+// from deploy.properties), either a NumberFormatException or
+// a ParseException would occur; even when the value...
[truncated message content] |
|
From: <btm...@us...> - 2010-12-06 22:09:50
|
Revision: 3997
http://bigdata.svn.sourceforge.net/bigdata/?rev=3997&view=rev
Author: btmurphy
Date: 2010-12-06 22:09:42 +0000 (Mon, 06 Dec 2010)
Log Message:
-----------
[branch dev-btm]: CHECKPOINT - changes to allow smart proxy wrapper for zookeeper to be started by ServicesManagerService; also changes to allow TestServiceStarter test to start smart proxy zookeeper [more fixes to junit tests needed]
Modified Paths:
--------------
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServicesManagerStartupTask.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServicesManagerConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ZookeeperServerConfiguration.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/ZookeeperProcessHelper.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/ServiceImpl.java
branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/JiniServicesHelper.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/AbstractFedZooTestCase.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/TestServiceStarter.java
branches/dev-btm/bigdata-jini/src/test/com/bigdata/zookeeper/AbstractZooTestCase.java
branches/dev-btm/src/resources/config/bigdataCluster.config
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServicesManagerStartupTask.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServicesManagerStartupTask.java 2010-12-06 21:29:19 UTC (rev 3996)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/ServicesManagerStartupTask.java 2010-12-06 22:09:42 UTC (rev 3997)
@@ -141,7 +141,12 @@
if (serviceConfig instanceof ZookeeperServerConfiguration) {
System.out.println("\n---- ServicesManagerStartupTask.doStartup: startZookeeperService() ----");
- startZookeeperService(config);
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - BEGIN
+//BTM - PRE_ZOOKEEPER_SMART_PROXY startZookeeperService(config);
+ startZookeeperService
+ ( ((ZookeeperServerConfiguration)serviceConfig).classType,
+ config );
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - END
}
@@ -227,23 +232,38 @@
*
* @return <code>true</code> if an instance was started successfully.
*/
- protected boolean startZookeeperService(final Configuration config)
- throws ConfigurationException, IOException {
-
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - BEGIN
+//BTM - PRE_ZOOKEEPER_SMART_PROXY protected boolean startZookeeperService(final Configuration config)
+//BTM - PRE_ZOOKEEPER_SMART_PROXY throws ConfigurationException, IOException {
+//BTM - PRE_ZOOKEEPER_SMART_PROXY
+//BTM - PRE_ZOOKEEPER_SMART_PROXY try {
+//BTM - PRE_ZOOKEEPER_SMART_PROXY
+//BTM - PRE_ZOOKEEPER_SMART_PROXY return ZookeeperProcessHelper.startZookeeper(config, service) > 0;
+//BTM - PRE_ZOOKEEPER_SMART_PROXY
+//BTM - PRE_ZOOKEEPER_SMART_PROXY } catch (Throwable t) {
+//BTM - PRE_ZOOKEEPER_SMART_PROXY
+//BTM - PRE_ZOOKEEPER_SMART_PROXY log.error(
+//BTM - PRE_ZOOKEEPER_SMART_PROXY "Could not start zookeeper service: " + t, t);
+//BTM - PRE_ZOOKEEPER_SMART_PROXY
+//BTM - PRE_ZOOKEEPER_SMART_PROXY return false;
+//BTM - PRE_ZOOKEEPER_SMART_PROXY
+//BTM - PRE_ZOOKEEPER_SMART_PROXY }
+//BTM - PRE_ZOOKEEPER_SMART_PROXY
+//BTM - PRE_ZOOKEEPER_SMART_PROXY }
+//BTM - PRE_ZOOKEEPER_SMART_PROXY
+ protected boolean startZookeeperService
+ (Class classType, Configuration config)
+ throws ConfigurationException, IOException
+ {
try {
-
- return ZookeeperProcessHelper.startZookeeper(config, service) > 0;
-
+ return ZookeeperProcessHelper.startZookeeper
+ (classType, config, service) > 0;
} catch (Throwable t) {
-
- log.error(
- "Could not start zookeeper service: " + t, t);
-
+ log.error("Could not start zookeeper service: " + t, t);
return false;
-
}
-
}
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - END
/**
* If necessary, start the jini core services on this host.
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServicesManagerConfiguration.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServicesManagerConfiguration.java 2010-12-06 21:29:19 UTC (rev 3996)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ServicesManagerConfiguration.java 2010-12-06 22:09:42 UTC (rev 3997)
@@ -225,7 +225,7 @@
} else if (a.equals(QuorumPeerMain.class.getName())) {
System.out.println("\n*** ServicesManagerConfiguration.getConfigurations: QuorumPeerMain BEGIN");
- v.add(new ZookeeperServerConfiguration(config));
+ v.add(new ZookeeperServerConfiguration(QuorumPeerMain.class, config));
System.out.println("*** ServicesManagerConfiguration.getConfigurations: QuorumPeerMain END\n");
} else if (a.equals(TransactionServer.class.getName())) {
@@ -261,6 +261,12 @@
//BTM - BEGIN: smart proxy impls ------------------------------------------------------------
+ } else if (a.equals(com.bigdata.quorum.ServiceImpl.class.getName())) {
+
+System.out.println("\n*** ServicesManagerConfiguration.getConfigurations: com.bigdata.quorum.ServiceImpl BEGIN");
+ v.add(new ZookeeperServerConfiguration(com.bigdata.quorum.ServiceImpl.class, config));
+System.out.println("*** ServicesManagerConfiguration.getConfigurations: com.bigdata.quorum.ServiceImpl END\n");
+
} else if (a.equals(com.bigdata.transaction.ServiceImpl.class.getName())) {//transaction service
System.out.println("\n*** ServicesManagerConfiguration.getConfigurations: com.bigdata.transaction.ServiceImpl BEGIN");
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ZookeeperServerConfiguration.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ZookeeperServerConfiguration.java 2010-12-06 21:29:19 UTC (rev 3996)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/config/ZookeeperServerConfiguration.java 2010-12-06 22:09:42 UTC (rev 3997)
@@ -257,15 +257,32 @@
}
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - BEGIN
+ public Class classType;
+ private Configuration jiniConfig;
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - END
+
/**
* @param className
* @param config
* @throws ConfigurationException
*/
- public ZookeeperServerConfiguration(final Configuration config)
- throws ConfigurationException {
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - BEGIN
+//BTM - PRE_ZOOKEEPER_SMART_PROXY public ZookeeperServerConfiguration(final Configuration config)
+//BTM - PRE_ZOOKEEPER_SMART_PROXY throws ConfigurationException {
+//BTM - PRE_ZOOKEEPER_SMART_PROXY
+//BTM - PRE_ZOOKEEPER_SMART_PROXY super(QuorumPeerMain.class.getName(), config);
+ public ZookeeperServerConfiguration(Class classType,
+ Configuration config)
+ throws ConfigurationException
+ {
- super(QuorumPeerMain.class.getName(), config);
+ super(classType.getName(), config);
+ this.classType = classType;
+ this.jiniConfig = config;
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - END
+//BTM
+System.out.println("*** ZookeeperServerConfiguration: constructor ***");
servers = (String) config.getEntry(Options.NAMESPACE, Options.SERVERS,
String.class);
@@ -455,6 +472,7 @@
public ZookeeperServiceStarter newServiceStarter(IServiceListener listener,
ZookeeperServerEntry entry) {
+System.out.println("*** ZookeeperServerConfiguration ---> newServiceStarter ***");
return new ZookeeperServiceStarter(listener, entry);
}
@@ -511,6 +529,7 @@
ZookeeperServerEntry entry) {
super(listener);
+System.out.println("*** ZookeeperServerConfiguration.ZookeeperServiceStarter: constructor - BEGIN ***");
if (entry == null)
throw new IllegalArgumentException();
@@ -524,6 +543,7 @@
*/
dataDir = new File(ZookeeperServerConfiguration.this.dataDir,
Integer.toString(entry.id)).getAbsoluteFile();
+System.out.println("*** ZookeeperServerConfiguration.ZookeeperServiceStarter: dataDir = "+dataDir);
if (log.isInfoEnabled())
log.info(Options.DATA_DIR + "=" + dataDir);
@@ -536,6 +556,7 @@
*/
dataLogDir = new File(ZookeeperServerConfiguration.this.dataLogDir,
Integer.toString(entry.id)).getAbsoluteFile();
+System.out.println("*** ZookeeperServerConfiguration.ZookeeperServiceStarter: dataLogDir = "+dataLogDir);
if (log.isInfoEnabled())
log.info(Options.DATA_LOG_DIR + "=" + dataLogDir);
@@ -544,9 +565,12 @@
configFile = new File(dataDir,
ZookeeperServerConfiguration.this.configFile)
.getAbsoluteFile();
+System.out.println("*** ZookeeperServerConfiguration.ZookeeperServiceStarter: generated configFile = "+configFile);
// the server id is written on this file.
myidFile = new File(dataDir, "myid");
+System.out.println("*** ZookeeperServerConfiguration.ZookeeperServiceStarter: myidFile = "+myidFile);
+System.out.println("*** ZookeeperServerConfiguration.ZookeeperServiceStarter: constructor - END ***");
}
@@ -618,7 +642,14 @@
/*
* Write the zookeeper server configuration file.
*/
- writeZookeeperConfigFile();
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - BEGIN
+//BTM - PRE_ZOOKEEPER_SMART_PROXY writeZookeeperConfigFile();
+ if ( (Options.NAMESPACE).equals(classType.getName()) ) {
+ writeZookeeperConfigFile();
+ } else {
+ writeConfigFile();// Jini config file
+ }
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - END
/*
* Start the server.
@@ -682,7 +713,9 @@
// the configuration file.
cmds.add(configFile.toString());
-
+//BTM - FOR_ZOOKEEPER_SMART_PROXY - BEGIN
+ cmds.add("com.bigdata.quorum."+getGroups("groupsToJoin"));
+//BTM - FOR_ZOOKEEPER_SMART_PROXY - END
}
@Override
@@ -859,6 +892,113 @@
}
+//BTM - FOR_ZOOKEEPER_SMART_PROXY - BEGIN ------------------------------------
+ protected void writeConfigFile() throws IOException {
+
+ // 1. generate the contents to be written to the file
+ final String contents;
+ {
+ final StringWriter out = new StringWriter();
+ writeConfigFile(out);
+ out.flush();
+ contents = out.toString();
+ }
+
+ // 2. write the contents generated above to the file.
+ {
+ final Writer out2 =
+ new OutputStreamWriter
+ (new BufferedOutputStream
+ (new FileOutputStream(configFile)));
+ try {
+ out2.write(contents);
+ out2.flush();
+ } finally {
+ out2.close();
+ }
+ }
+ }
+
+ protected void writeConfigFile(final Writer out) throws IOException {
+ // 1. Comments
+ writeComments(out);
+ out.write("\n");
+
+ // 2. import statements
+ for (String i : getImports()) {// write import statements
+ out.write("import " + i + ";\n");
+ }
+ out.write("\n");
+
+ // 3. Open component name
+ out.write("\n\n" + classType.getPackage().getName() + " {\n");
+ {
+ out.write(" "+Options.DATA_DIR+" = new File("+"\""+dataDir.toString()+"\""+");\n");
+ out.write(" "+Options.DATA_LOG_DIR+" = new File("+"\""+dataLogDir.toString()+"\""+");\n");
+ out.write(" "+Options.CLIENT_PORT+" = "+clientPort+";\n");
+ out.write(" "+Options.SERVERS+" = "+"\""+servers+"\""+";\n");
+ for(Map.Entry<String, String> entry : other.entrySet()) {
+ String key = entry.getKey();
+ String val = entry.getValue();
+ out.write(" "+key+" = "+val+";\n");
+ }
+
+ out.write(" "+getGroups("groupsToJoin")+";\n");
+ out.write(" locsToJoin=new LookupLocator[]{ };\n");
+ out.write(" static discoveryManager = new LookupDiscoveryManager(groupsToJoin,locsToJoin,null,this);\n");
+ }// END BLOCK - 4. Close component name
+ out.write("}\n");
+ }
+
+ protected void writeComments(final Writer out) throws IOException {
+ out.write("// className=" + className + "\n");
+ out.write("// date=" + new java.util.Date() + "\n");
+ }
+
+ public String[] getImports() {
+ return new String[]
+ { "java.net.NetworkInterface",
+ "com.sun.jini.config.ConfigUtil",
+ "net.jini.constraint.BasicMethodConstraints",
+ "net.jini.core.constraint.ConnectionRelativeTime",
+ "net.jini.core.constraint.InvocationConstraints",
+ "net.jini.jeri.BasicILFactory",
+ "net.jini.jeri.BasicJeriExporter",
+ "net.jini.jeri.tcp.TcpServerEndpoint",
+ "net.jini.core.discovery.LookupLocator",
+ "net.jini.discovery.LookupDiscoveryManager",
+ "com.bigdata.util.config.NicUtil",
+ "com.bigdata.util.config.ConfigDeployUtil"
+ };
+ }
+
+ protected String getGroups(String entryName) {
+ String fedname = null;
+ String zrootname = null;//in some test configs
+ String defaultGroupName = "UNKNOWN";
+ try {
+ fedname = (String) jiniConfig.getEntry
+ ("bigdata", "fedname",
+ String.class, "fedname-unset");
+ zrootname = (String) jiniConfig.getEntry
+ ("bigdata", "zrootname",
+ String.class, null);
+ } catch(Exception e) {//swallow
+ }
+ if (fedname == null) fedname = defaultGroupName;
+
+ StringBuffer strBuf =
+ new StringBuffer(entryName+"=new String[]{");
+ strBuf.append( com.bigdata.jini.util.ConfigMath.q(fedname) );
+ if (zrootname != null) {
+ strBuf.append
+ (","+com.bigdata.jini.util.ConfigMath.q(zrootname));
+ }
+ strBuf.append("}");
+
+ return strBuf.toString();
+ }
+//BTM - FOR_ZOOKEEPER_SMART_PROXY - END --------------------------------------
}
/**
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/ZookeeperProcessHelper.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/ZookeeperProcessHelper.java 2010-12-06 21:29:19 UTC (rev 3996)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/jini/start/process/ZookeeperProcessHelper.java 2010-12-06 22:09:42 UTC (rev 3997)
@@ -147,13 +147,23 @@
*
* @see ZookeeperServerConfiguration#newServiceStarter(IServiceListener, ZookeeperServerEntry)
*/
- static public int startZookeeper(final Configuration config,
- final IServiceListener listener) throws ConfigurationException,
- IOException {
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - BEGIN
+//BTM - PRE_ZOOKEEPER_SMART_PROXY static public int startZookeeper(final Configuration config,
+//BTM - PRE_ZOOKEEPER_SMART_PROXY final IServiceListener listener) throws ConfigurationException,
+//BTM - PRE_ZOOKEEPER_SMART_PROXY IOException {
+//BTM - PRE_ZOOKEEPER_SMART_PROXY
+//BTM - PRE_ZOOKEEPER_SMART_PROXY final ZookeeperServerConfiguration serverConfig = new ZookeeperServerConfiguration(
+//BTM - PRE_ZOOKEEPER_SMART_PROXY config);
+ static public int startZookeeper
+ (final Class classType,
+ final Configuration config,
+ final IServiceListener listener)
+ throws ConfigurationException, IOException
+ {
+ final ZookeeperServerConfiguration serverConfig =
+ new ZookeeperServerConfiguration(classType, config);
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - END
- final ZookeeperServerConfiguration serverConfig = new ZookeeperServerConfiguration(
- config);
-
//BTM log.warn("\n------------ ZookeeperProcessHelper.startZookeeper: [localhost="+thisInetAddr.getHostName()+", clientPort="+serverConfig.clientPort+"]\n");
//BTM com.bigdata.util.Util.printStr("TestBigdata.debug","\n------------ ZookeeperProcessHelper.startZookeeper: [localhost="+thisInetAddr.getHostName()+", clientPort="+serverConfig.clientPort+"]\n");
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/ServiceImpl.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/ServiceImpl.java 2010-12-06 21:29:19 UTC (rev 3996)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/quorum/ServiceImpl.java 2010-12-06 22:09:42 UTC (rev 3997)
@@ -98,6 +98,7 @@
import java.net.SocketException;
import java.net.UnknownHostException;
import java.rmi.RemoteException;
+import java.rmi.RMISecurityManager;
import java.rmi.server.ExportException;
import java.util.ArrayList;
import java.util.Collections;
@@ -117,11 +118,20 @@
/**
* Backend (admin) zookeeper based implementation of the quorum peer service.
+ *
+ * Note: this class is currently declared public rather than the preferred
+ * package protected scope. This is so that the JiniServicesHelper
+ * utility can instantiate this class in the tests that are currently
+ * implemented to interact directly with the service's backend;
+ * as opposed to starting the service with the ServiceStarter and
+ * then interacting with the service through the discovered service
+ * frontend.
*/
+public
class ServiceImpl implements PrivateInterface {
- private static Logger logger = LogUtil.getLog4jLogger
- ( (ServiceImpl.class).getName() ) ;
+ private static Logger logger =
+ LogUtil.getLog4jLogger(COMPONENT_NAME);
private static String shutdownStr;
private static String killStr;
@@ -267,6 +277,9 @@
//Initialize the service from the config
private void init(String[] args) throws Exception {
+ if(System.getSecurityManager() == null) {
+ System.setSecurityManager(new RMISecurityManager());
+ }
config = ConfigurationProvider.getInstance
( args,
(this.getClass()).getClassLoader() );
@@ -1388,4 +1401,116 @@
}
}
}
+
+ /**
+ * This main() method is provided because it may be desirable (for
+ * testing or other reasons) to be able to start this service using
+ * a command line that is either manually entered in a command window
+ * or supplied to the java ProcessBuilder class for execution.
+ * <p>
+ * The mechanism that currently employs the ProcessBuilder class to
+ * execute a dynamically generated command line will be referred to
+ * as the 'ServiceConfiguration mechanism', which involves the use of
+ * the following ServiceConfiguration class hierarchy,
+ * <p>
+ * <ul>
+ * <li> ZookeeperConfiguration
+ * <li> JavaServiceConfiguration
+ * <li> ServiceConfiguration
+ * </ul>
+ * </p>
+ * The ServicesConfiguration mechanism may involve the use of the
+ * ServicesManagerService directly to execute this service, or it may
+ * involve the use of the junit framework to start this service. In
+ * either case, a command line is constructed from information that is
+ * specified at each of the various ServiceConfiguration levels, and
+ * is ultimately executed in a ProcessBuilder instance (in the
+ * ProcessHelper class).
+ * <p>
+ * In order for this method to know whether or not the
+ * ServiceConfiguration mechanism is being used to start the service,
+ * this method must be told that the ServiceConfiguration mechanism is
+ * being used. This is done by setting the system property named
+ * <code>usingServiceConfiguration</code> to any non-null value.
+ * <p>
+ * When the ServiceConfiguration mechanism is <i>not</i> used to start
+ * this service, this method assumes that the element at index 0
+ * of the args array references the path to the jini configuration
+ * file that will be input by this method to this service's constructor.
+ * On the other hand, when the ServiceConfiguration mechanism <i>is</i>
+ * used to start this service, the service's configuration is handled
+ * differently, as described below.
+ * <p>
+ * When using the ServiceConfiguration mechanism, in addition to
+ * generating a command line to start the service, although an initial,
+ * pre-constructed jini configuration file is supplied (to the
+ * ServicesManagerService or the test framework, for example), a
+ * second jini configuration file is generated <i>on the fly</i> as
+ * well. When generating that new configuration file, a subset of the
+ * components and entries specified in the initial jini configuration
+ * are retrieved and placed in the new configuration being generated.
+ * It is that second, newly-generated configuration file that is input
+ * to this method through the args elements at index 0.
+ * <p>
+ * When the ServiceConfiguration mechanism is used to invoke this
+ * method, this method makes a number of assumptions. One assumption
+ * is that there is a component with name equal to the value,
+ * "org.apache.zookeeper.server.quorum.QuorumPeerMain", as well as
+ * either a component itself, or entries corresponding to a component,
+ * with name equal to the fully qualified name of this class (or both).
+ * Another assumption is that an entry named 'args' is associated with
+ * that component. The 'args' entry is assumed to be a <code>String</code>
+ * array in which one of the elments is specified to be a system
+ * property named 'config' whose value is equal to the path and
+ * filename of yet a third jini configuration file; that is, something
+ * of the form, "-Dconfig=<path-to-another-jini-config>". It is this
+ * third jini configuration file that the service will ultimately use
+ * to initialize itself when the ServiceConfiguration mechanism is
+ * being used to start the service. In that case then, this method
+ * will retrieve the path to the third jini configuration file from
+ * the configuration file supplied to this method in the args array
+ * at index 0, and then replace the element at index 0 with that
+ * path; so that when the service is instantiated (using this class'
+ * constructor), that third configuration file is made available to
+ * the service instance.
+ * <p>
+ * Note that, unlike the other service implementations, this service
+ * always generates its own service id the very first time it is
+ * started, persists that service id, and retrieves and reuses it
+ * on restarts.
+ * <p>
+ * Note that once an instance of this service implementation class
+ * has been created, that instance is stored in the <code>thisImpl</code>
+ * field to prevent the instance from being garbage collected until
+ * the service is actually shutdown.
+ */
+
+ private static ServiceImpl thisImpl;
+
+ public static void main(String[] args) {
+ logger.debug("[main]: appHome="+System.getProperty("appHome"));
+ try {
+ // If the system property with name "config" is set, then
+ // use the value of that property to override the value
+ // input in the first element of the args array
+ ArrayList<String> argsList = new ArrayList<String>();
+ int begIndx = 0;
+ String configFile = System.getProperty("config");
+ if(configFile != null) {
+ // Replace args[0] with config file location
+ argsList.add(configFile);
+ begIndx = 1;
+ }
+ for(int i=begIndx; i<args.length; i++) {
+ argsList.add(args[i]);
+ }
+ logger.debug("[main]: instantiating service [new ServiceImpl]");
+ thisImpl = new ServiceImpl
+ ( argsList.toArray(new String[argsList.size()]),
+ new com.bigdata.service.jini.FakeLifeCycle() );
+ } catch(Throwable t) {
+ logger.log(Level.WARN,
+ "failed to start callable executor service", t);
+ }
+ }
}
Modified: branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/JiniServicesHelper.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/JiniServicesHelper.java 2010-12-06 21:29:19 UTC (rev 3996)
+++ branches/dev-btm/bigdata-jini/src/java/com/bigdata/service/jini/util/JiniServicesHelper.java 2010-12-06 22:09:42 UTC (rev 3997)
@@ -555,8 +555,14 @@
// start zookeeper (a server instance).
//BTM log.warn("\n---------------- JiniServicesHelper.innerStart >>> START ZOOKEEPER\n");
//BTM com.bigdata.util.Util.printStr("TestBigdata.debug","\n---------------- JiniServicesHelper.innerStart >>> START ZOOKEEPER\n");
- final int nstarted = ZookeeperProcessHelper.startZookeeper(
- config, serviceListener);
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - BEGIN
+//BTM - PRE_ZOOKEEPER_SMART_PROXY final int nstarted = ZookeeperProcessHelper.startZookeeper(
+//BTM - PRE_ZOOKEEPER_SMART_PROXY config, serviceListener);
+ final int nstarted =
+ ZookeeperProcessHelper.startZookeeper
+ (com.bigdata.quorum.ServiceImpl.class, //BTM - was QuorumPeerMain.class
+ config, serviceListener);
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - END
//BTM log.warn("\n---------------- JiniServicesHelper.innerStart >>> START ZOOKEEPER - DONE\n");
//BTM com.bigdata.util.Util.printStr("TestBigdata.debug","\n---------------- JiniServicesHelper.innerStart >>> START ZOOKEEPER - DONE\n");
Modified: branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/AbstractFedZooTestCase.java
===================================================================
--- branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/AbstractFedZooTestCase.java 2010-12-06 21:29:19 UTC (rev 3996)
+++ branches/dev-btm/bigdata-jini/src/test/com/bigdata/jini/start/AbstractFedZooTestCase.java 2010-12-06 22:09:42 UTC (rev 3997)
@@ -50,6 +50,16 @@
import com.bigdata.service.jini.JiniClient;
import com.bigdata.service.jini.JiniFederation;
+//BTM - FOR_ZOOKEEPER_SMART_PROXY - BEGIN
+import com.bigdata.service.QuorumPeerService;
+import com.sun.jini.admin.DestroyAdmin;
+import net.jini.admin.Administrable;
+import net.jini.core.lookup.ServiceItem;
+import net.jini.core.lookup.ServiceTemplate;
+import net.jini.discovery.DiscoveryGroupManagement;
+import net.jini.lookup.ServiceDiscoveryManager;
+//BTM - FOR_ZOOKEEPER_SMART_PROXY - END
+
/**
* Abstract base class for unit tests requiring a running zookeeper and a
* running federation as configured from a test resource.
@@ -143,7 +153,11 @@
//BTM - FOR_CLIENT_SERVICE - END
// if necessary, start zookeeper (a server instance).
- ZookeeperProcessHelper.startZookeeper(config, listener);
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - BEGIN
+//BTM - PRE_ZOOKEEPER_SMART_PROXY ZookeeperProcessHelper.startZookeeper(config, listener);
+ ZookeeperProcessHelper.startZookeeper(com.bigdata.quorum.ServiceImpl.class, config, listener);
+//ZookeeperProcessHelper.startZookeeper(org.apache.zookeeper.server.quorum.QuorumPeerMain.class, config, listener);
+//BTM - PRE_ZOOKEEPER_SMART_PROXY - END
/*
* FIXME We need to start a jini lookup service for groups = {fedname}
@@ -171,6 +185,29 @@
System.err.println(getName() + ": tearing down zrootname=" + zrootname);
+//BTM - FOR_ZOOKEEPER_SMART_PROXY - BEGIN
+ // Graceful shutdown of QuorumPeerService
+ ServiceDiscoveryManager sdm = fed.getServiceDiscoveryManager();
+ DiscoveryGroupManagement ldm =
+ (DiscoveryGroupManagement)(sdm.getDiscoveryManager());
+ Class[] quorumServiceType =
+ new Class[] {com.bigdata.service.QuorumPeerService.class};
+ ServiceTemplate quorumServiceTmpl =
+ new ServiceTemplate(null, quorumServiceType, null);
+ ServiceItem[] items =
+ sdm.lookup(q...
[truncated message content] |