Menu
▾
▴
[cweb-CVS] bigdata/src/java/com/bigdata/service AbstractClient.java, NONE, 1.1 MetadataServer.java, NONE, 1.1 IMetadataService.java, 1.2, 1.3 AbstractServer.java, 1.4, 1.5 DataServer.java, 1.3, 1.4 MetadataService.java, 1.2, 1.3 TransactionService.java, 1.2, 1.3 IDataService.java, 1.3, 1.4 DataServiceClient.java, 1.3, 1.4
|
From: Bryan T. <tho...@us...> - 2007-03-27 14:34:33
|
Update of /cvsroot/cweb/bigdata/src/java/com/bigdata/service In directory sc8-pr-cvs4.sourceforge.net:/tmp/cvs-serv6186/src/java/com/bigdata/service Modified Files: IMetadataService.java AbstractServer.java DataServer.java MetadataService.java TransactionService.java IDataService.java DataServiceClient.java Added Files: AbstractClient.java MetadataServer.java Log Message: Added indexUUID to AbstractBTree so that each scale-out index may have a unique indentifier. Modified the BTreeMetadata class and derived classes to use Externalizable, to support explicit versioning of the metadata record, and to have private fields since they can not be final with Externalizable. Index: DataServer.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/service/DataServer.java,v retrieving revision 1.3 retrieving revision 1.4 diff -C2 -d -r1.3 -r1.4 *** DataServer.java 23 Mar 2007 20:01:25 -0000 1.3 --- DataServer.java 27 Mar 2007 14:34:23 -0000 1.4 *************** *** 52,55 **** --- 52,56 ---- import java.util.Properties; + import com.bigdata.journal.IJournal; import com.sun.jini.start.LifeCycle; *************** *** 100,103 **** --- 101,132 ---- /** + * Extends the behavior to close and delete the journal in use by the data + * service. + */ + protected void destroy() { + + DataService service = (DataService)impl; + + super.destroy(); + + try { + + IJournal journal = service.journal; + + log.info("Closing and deleting: "+journal.getFile()); + + journal.closeAndDelete(); + + log.info("Journal deleted."); + + } catch (Throwable t) { + + log.warn("Could not delete journal: " + t, t); + + } + + } + + /** * Adds jini administration interfaces to the basic {@link DataService}. * *************** *** 144,179 **** public void run() { ! server.shutdownNow(); ! ! log.info("Deleting state."); - try { - - journal.closeAndDelete(); - - log.info("Journal deleted."); - - } catch (Throwable t) { - - log.warn("Could not delete journal: " + t, t); - - } - - if (!server.serviceIdFile.delete()) { - - log.warn("Could not delete file: " - + server.serviceIdFile); - - } - - try { - Thread.sleep(3); - } catch (InterruptedException ex) { - } - log.info("Service stopped."); - System.exit(1); - } --- 173,180 ---- public void run() { ! server.destroy(); log.info("Service stopped."); } Index: IDataService.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/service/IDataService.java,v retrieving revision 1.3 retrieving revision 1.4 diff -C2 -d -r1.3 -r1.4 *** IDataService.java 22 Mar 2007 21:11:24 -0000 1.3 --- IDataService.java 27 Mar 2007 14:34:24 -0000 1.4 *************** *** 82,85 **** --- 82,92 ---- * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ + * + * @todo add support for triggers. unisolated triggers must be asynchronous if + * they will take actions with high latency (such as writing on a + * different index partition, which could be remote). Low latency actions + * might include emitting asynchronous messages. transactional triggers + * can have more flexibility since they are under less of a latency + * constraint. */ public interface IDataService extends IRemoteTxCommitProtocol { Index: DataServiceClient.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/service/DataServiceClient.java,v retrieving revision 1.3 retrieving revision 1.4 diff -C2 -d -r1.3 -r1.4 *** DataServiceClient.java 22 Mar 2007 21:11:24 -0000 1.3 --- DataServiceClient.java 27 Mar 2007 14:34:24 -0000 1.4 *************** *** 84,96 **** } - - /* - * @todo implement remote data service client talking to NIO service - * instance. this needs to locate the transaction manager service and - * the metadata service for each index used by the client. - */ - abstract private class NIODataServiceClient implements IDataService { - - } /** --- 84,87 ---- *************** *** 118,125 **** } - // public void map(long tx, String name, byte[] fromKey, byte[] toKey, IMapOp op) throws InterruptedException, ExecutionException { - // delegate.map(tx, name, fromKey, toKey, op); - // } - public RangeQueryResult rangeQuery(long tx, String name, byte[] fromKey, byte[] toKey, int flags) throws InterruptedException, ExecutionException, IOException { return delegate.rangeQuery(tx, name, fromKey, toKey, flags); --- 109,112 ---- --- NEW FILE: AbstractClient.java --- /** The Notice below must appear in each file of the Source Code of any copy you distribute of the Licensed Product. Contributors to any Modifications may add their own copyright notices to identify their own contributions. License: The contents of this file are subject to the CognitiveWeb Open Source License Version 1.1 (the License). You may not copy or use this file, in either source code or executable form, except in compliance with the License. You may obtain a copy of the License from http://www.CognitiveWeb.org/legal/license/ Software distributed under the License is distributed on an AS IS basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. Copyrights: Portions created by or assigned to CognitiveWeb are Copyright (c) 2003-2003 CognitiveWeb. All Rights Reserved. Contact information for CognitiveWeb is available at http://www.CognitiveWeb.org Portions Copyright (c) 2002-2003 Bryan Thompson. Acknowledgements: Special thanks to the developers of the Jabber Open Source License 1.0 (JOSL), from which this License was derived. This License contains terms that differ from JOSL. Special thanks to the CognitiveWeb Open Source Contributors for their suggestions and support of the Cognitive Web. Modifications: */ /* * Created on Mar 24, 2007 */ package com.bigdata.service; import java.io.IOException; import java.rmi.Remote; import net.jini.config.Configuration; import net.jini.config.ConfigurationException; import net.jini.config.ConfigurationProvider; import net.jini.core.discovery.LookupLocator; import net.jini.core.lookup.ServiceRegistrar; import net.jini.core.lookup.ServiceTemplate; import net.jini.discovery.DiscoveryEvent; import net.jini.discovery.DiscoveryListener; import net.jini.discovery.DiscoveryManagement; import net.jini.discovery.LookupDiscovery; import net.jini.discovery.LookupDiscoveryManager; import org.apache.log4j.Logger; import com.bigdata.journal.ITransactionManager; /** * Abstract base class for a bigdata client. * <p> * Clients are configured to perform service lookup with a jini group that * identifies the bigdata federation. Clients begin by discovering the * {@link IMetadataService}. Clients use the {@link IMetadataService} to manage * indices (add/drop/proxy). Once a client has a proxy for an index, it carries * out read and write operations using that proxy. The proxy is responsible for * transparently discovering the {@link IDataService}s on which the index * partitions are located and directing read and write operations appropriately. * <p> * A client may discover and use an {@link ITransactionManager} if needs to use * transactions as opposed to unisolated reads and writes. When the client * requests a transaction, the transaction manager responds with a long integer * containing the transaction identifier - this is simply the unique start time * assigned to that transaction by the transaction manager. The client then * provides that transaction identifier for operations that are isolated within * the transaction. When the client is done with the transaction, it must use * the transaction manager to either abort or commit the transaction. * (Transactions that fail to progress may be eventually aborted.) * <p> * When using unisolated operations, the client does not need to resolve or use * the transaction manager and it simply specifies <code>0L</code> as the * transaction identifier for its read and write operations. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ */ public class AbstractClient implements DiscoveryListener { public static final transient Logger log = Logger .getLogger(AbstractClient.class); /** * The label in the {@link Configuration} file for the service * description. */ protected final static transient String SERVICE_LABEL = "ServiceDescription"; /** * The label in the {@link Configuration} file for the service advertisment * data. */ protected final static transient String ADVERT_LABEL = "AdvertDescription"; private DiscoveryManagement discoveryManager; private Configuration config; /** * The exported proxy for the service implementation object. */ protected Remote proxy; /** * Server startup reads {@link Configuration} data from the file(s) named by * <i>args</i>, starts the service, and advertises the service for * discovery. Aside from the server class to start, the behavior is more or * less entirely parameterized by the {@link Configuration}. * * @param args * The command line arguments. */ protected AbstractClient(String[] args) { // @todo verify that this belongs here. System.setSecurityManager(new SecurityManager()); LookupLocator[] unicastLocators = null; String[] groups = null; try { config = ConfigurationProvider.getInstance(args); /* * Extract how the client will discover services from the * Configuration. */ groups = (String[]) config.getEntry(ADVERT_LABEL, "groups", String[].class, LookupDiscovery.ALL_GROUPS/* default */); unicastLocators = (LookupLocator[]) config .getEntry(ADVERT_LABEL, "unicastLocators", LookupLocator[].class, null/* default */); } catch (ConfigurationException ex) { log.fatal("Configuration error: " + ex, ex); System.exit(1); } try { /* * Note: This class will perform multicast discovery if ALL_GROUPS * is specified and otherwise requires you to specify one or more * unicast locators (URIs of hosts running discovery services). As * an alternative, you can use LookupDiscovery, which always does * multicast discovery. */ discoveryManager = new LookupDiscoveryManager(groups, unicastLocators, this ); // discoveryManager = new LookupDiscovery(groups); } catch (IOException ex) { log.fatal("Lookup service discovery error: " + ex, ex); try { discoveryManager.terminate(); } catch (Throwable t) { /* ignore */ } System.exit(1); } } /** * Return the data service matched on this registrar. * * @param registrar * * @return The data service or <code>null</code> if none was matched. * * @todo this belongs in the metadata service since it needs to discover * data services. It also needs to know when data services start and * stop so it needs updates based on the service template. * * @todo the client on the other hand needs to discover a single metadata * service and a single transaction manager service. if either the * metadata service or the transaction manager service goes down, then * it needs to discover another service so that it can keep working. * * @todo we need to describe the services to be discovered by their primary * interface and only search within a designated group that * corresponds to the bigdata federation of interest - that group is * part of the client configuration. */ public IDataService getDataService(ServiceRegistrar registrar) { Class[] classes = new Class[] {IDataService.class}; ServiceTemplate template = new ServiceTemplate(null, classes, null); IDataService proxy = null; try { proxy = (IDataService) registrar.lookup(template); } catch(java.rmi.RemoteException e) { log.warn(e); } return proxy; } /** * Return an {@link IMetadataService}. * * @param registrar * A service registrar to query. * * @return An {@link IMetadataService} if one was found using that * registrar. */ public IMetadataService getMetadataService(ServiceRegistrar registrar) { Class[] classes = new Class[] {IMetadataService.class}; ServiceTemplate template = new ServiceTemplate(null, classes, null); IMetadataService proxy = null; try { proxy = (IMetadataService) registrar.lookup(template); } catch(java.rmi.RemoteException e) { log.warn(e); } return proxy; } /** * Return an {@link IMetadataService}. * * @param a * An array of registrars to query. * * @return An {@link IMetadataService} if one was found. * * @todo while the client only needs a single metadata service, the data * services themselves must register with all metadata services * discovered in their group (and I must sort out how the determine * primary vs secondary metadata services, e.g., by a status on the * service or some custom api). */ public IMetadataService getMetadataService(ServiceRegistrar[] a) { IMetadataService proxy = null; for(int i=0; i<a.length && proxy == null; i++) { proxy = getMetadataService(a[i]); } return proxy; } /** * Invoked when a lookup service is discarded. */ public void discarded(DiscoveryEvent arg0) { log.info(""+arg0); // TODO Auto-generated method stub } /** * Invoked when a lookup service is discovered. */ public void discovered(DiscoveryEvent arg0) { log.info(""+arg0); // TODO Auto-generated method stub } } Index: TransactionService.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/service/TransactionService.java,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** TransactionService.java 17 Mar 2007 23:14:58 -0000 1.2 --- TransactionService.java 27 Mar 2007 14:34:23 -0000 1.3 *************** *** 97,100 **** --- 97,108 ---- * @todo track ground states so that we known when we can release old journals * and index segments? + * + * @todo the transactional model might include a counter for the #of clients + * that have started work on a transaction in order to support distributed + * start/commit protocols. if clients use a workflow model, then they + * could pass the responsibility for the counter along with the + * transaction identifier rather than decrementing the counter themselves. + * It might be good to be able to identify which clients are still working + * on a given transaction. */ public class TransactionService implements ITransactionManager, IServiceShutdown { Index: MetadataService.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/service/MetadataService.java,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** MetadataService.java 22 Mar 2007 15:04:15 -0000 1.2 --- MetadataService.java 27 Mar 2007 14:34:23 -0000 1.3 *************** *** 60,63 **** --- 60,89 ---- * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ + * + * FIXME Tag each index with a UUID. The UUID needs to appear in the index + * metadata record for each journal and index segment. When it is an named + * (scale-out) index, the UUID of the scale-out index must be used for each + * B+Tree metadata record having data for that index. This allows us to map + * backwards from the data structures to the metadata index. Document this in + * the UML model. (I still need to get the correct index UUID to each BTree + * constuctor since they are all using a Random UUID right now.) + * + * @todo Provide a means to reconstruct the metadata index from the journal and + * index segment data files. We tag each journal and index segment with a + * UUID. Each index is also tagged with a UUID, and that UUID is written + * into the metadata record for the index on each journal and index + * segment. Based on those UUIDs we are able to work backwards from the + * data on disk and identify the indices to which they belong. That + * information in combination with the timestamps in the metadata records + * and the first/last keys in the index partition is sufficient to + * regenerate the metadata indices. + * + * @todo A temporal/immortable database can be realized if we never delete old + * journals since they contain the historical committed states of the + * database. The use of index segments would still provide fast read + * performance on recent data, while a suitable twist on the metadata + * index would allow access to those historical states. (E.g., you have to + * be able to access the historical state of the metadata index that + * corresponds to the commit time of interest for the database.) */ public class MetadataService implements IMetadataService, IServiceShutdown { *************** *** 69,73 **** * {@link MetadataIndex} and {@link MasterJournal}. */ ! protected final Journal journal; public MetadataService(Properties properties) { --- 95,99 ---- * {@link MetadataIndex} and {@link MasterJournal}. */ ! protected Journal journal; public MetadataService(Properties properties) { *************** *** 78,89 **** */ ! throw new UnsupportedOperationException(); ! } - public static void main(String[] args) { - - } - public InetSocketAddress getDataService(String name,byte[] key) { // TODO Auto-generated method stub --- 104,111 ---- */ ! journal = new Journal(properties); ! } public InetSocketAddress getDataService(String name,byte[] key) { // TODO Auto-generated method stub --- NEW FILE: MetadataServer.java --- /** The Notice below must appear in each file of the Source Code of any copy you distribute of the Licensed Product. Contributors to any Modifications may add their own copyright notices to identify their own contributions. License: The contents of this file are subject to the CognitiveWeb Open Source License Version 1.1 (the License). You may not copy or use this file, in either source code or executable form, except in compliance with the License. You may obtain a copy of the License from http://www.CognitiveWeb.org/legal/license/ Software distributed under the License is distributed on an AS IS basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. Copyrights: Portions created by or assigned to CognitiveWeb are Copyright (c) 2003-2003 CognitiveWeb. All Rights Reserved. Contact information for CognitiveWeb is available at http://www.CognitiveWeb.org Portions Copyright (c) 2002-2003 Bryan Thompson. Acknowledgements: Special thanks to the developers of the Jabber Open Source License 1.0 (JOSL), from which this License was derived. This License contains terms that differ from JOSL. Special thanks to the CognitiveWeb Open Source Contributors for their suggestions and support of the Cognitive Web. Modifications: */ /* * Created on Mar 24, 2007 */ package com.bigdata.service; import java.rmi.Remote; import java.rmi.RemoteException; import java.util.Properties; import net.jini.core.lookup.ServiceMatches; import net.jini.core.lookup.ServiceRegistrar; import net.jini.core.lookup.ServiceTemplate; import com.bigdata.journal.IJournal; import com.sun.jini.start.LifeCycle; /** * A metadata server. * <p> * The metadata server is used to manage the life cycles of scale-out indices * and exposes proxies for read and write operations on indices to clients. * Clients use index proxies, which automatically direct reads and writes to the * {@link IDataService} on which specific index partitions are located. * <p> * On startup, the metadata service discovers active data services configured in * the same group. While running, it tracks when data services start and stop so * that it can (re-)allocate index partitions as necessary. * <p> * The metadata server uses a write through pipeline to replicate its data onto * registered secondary metadata servers. If the metadata server fails, clients * will automatically fail over to a secondary metadata server. Only the primary * metadata server actively tracks the state of data services since secondaries * are updated via the write through pipeline to ensure consistency. Secondary * metadata servers will notice if the primary dies and elect a new master. * * @todo note that the service update registration is _persistent_ (assuming * that the service registrar is persistent I suppose) so that will add a * wrinkle to how a bigdata instance must be configured. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ */ public class MetadataServer extends AbstractServer { /** * @param args */ public MetadataServer(String[] args) { super(args); } /** * @param args * @param lifeCycle */ public MetadataServer(String[] args, LifeCycle lifeCycle) { super(args, lifeCycle); } protected Remote newService(Properties properties) { return new AdministrableMetadataService(this,properties); } /** * Return an {@link IMetadataService}. * * @param registrar * A service registrar to query. * * @return An {@link IMetadataService} if one was found using that * registrar. */ public IMetadataService getMetadataService(ServiceRegistrar registrar) { Class[] classes = new Class[] {IMetadataService.class}; ServiceTemplate template = new ServiceTemplate(null, classes, null); IMetadataService proxy = null; try { proxy = (IMetadataService) registrar.lookup(template); } catch(java.rmi.RemoteException e) { log.warn(e); } return proxy; } /** * Return the data service(s) matched on this registrar. * * @param registrar * * @return The data service or <code>null</code> if none was matched. * * @todo we need to describe the services to be discovered by their primary * interface and only search within a designated group that * corresponds to the bigdata federation of interest - that group is * part of the client configuration. * * @todo how do we ensure that we have seen all data services? If we query * each registrar as it is discovered and then register for updates * there are two ways in which we could miss some instances: (1) new * data services register between the query and the registration for * updates; and (2) the query will not return _ALL_ data services * registered, but only as match as the match limit. */ public ServiceMatches getDataServices(ServiceRegistrar registrar) { Class[] classes = new Class[] {IDataService.class}; ServiceTemplate template = new ServiceTemplate(null, classes, null); try { return registrar.lookup(template,0); } catch(java.rmi.RemoteException e) { log.warn(e); return null; } } /** * Extends the behavior to close and delete the journal in use by the * metadata service. */ protected void destroy() { MetadataService service = (MetadataService)impl; super.destroy(); try { IJournal journal = service.journal; log.info("Closing and deleting: "+journal.getFile()); journal.closeAndDelete(); log.info("Journal deleted."); } catch (Throwable t) { log.warn("Could not delete journal: " + t, t); } } /** * Adds jini administration interfaces to the basic {@link MetadataService}. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ */ protected static class AdministrableMetadataService extends MetadataService implements Remote, RemoteAdministrable, RemoteDestroyAdmin { protected AbstractServer server; /** * @param properties */ public AdministrableMetadataService(AbstractServer server, Properties properties) { super(properties); this.server = server; } public Object getAdmin() throws RemoteException { log.info(""); return server.proxy; } /* * DestroyAdmin */ /** * Destroy the service and deletes any files containing resources (<em>application data</em>) * that was in use by that service. * * @throws RemoteException */ public void destroy() throws RemoteException { log.info(""); new Thread() { public void run() { server.destroy(); log.info("Service stopped."); } }.start(); } } } Index: AbstractServer.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/service/AbstractServer.java,v retrieving revision 1.4 retrieving revision 1.5 diff -C2 -d -r1.4 -r1.5 *** AbstractServer.java 23 Mar 2007 20:01:25 -0000 1.4 --- AbstractServer.java 27 Mar 2007 14:34:23 -0000 1.5 *************** *** 178,182 **** * The service implementation object. */ ! private Remote impl; /** * The exported proxy for the service implementation object. --- 178,182 ---- * The service implementation object. */ ! protected Remote impl; /** * The exported proxy for the service implementation object. *************** *** 640,643 **** --- 640,666 ---- } + + /** + * Contract is to shutdown the services and <em>destroys</em> its + * persistent state. This implementation calls {@link #shutdownNow()} and + * then deletes the {@link #serviceIdFile}. + * <p> + * Concrete subclasses SHOULD extend this method to destroy their persistent + * state. + */ + protected void destroy() { + + shutdownNow(); + + log.info("Deleting: "+serviceIdFile); + + if (!serviceIdFile.delete()) { + + log.warn("Could not delete file: " + + serviceIdFile); + + } + + } // /** Index: IMetadataService.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/service/IMetadataService.java,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** IMetadataService.java 22 Mar 2007 15:04:15 -0000 1.2 --- IMetadataService.java 27 Mar 2007 14:34:23 -0000 1.3 *************** *** 48,52 **** --- 48,54 ---- package com.bigdata.service; + import java.io.IOException; import java.net.InetSocketAddress; + import java.rmi.Remote; /** *************** *** 54,70 **** * <p> * The metadata service maintains locator information for the data service ! * instances responsible for each partition in the named index. Partitions ! * are automatically split when they overflow (~200M) and joined when they ! * underflow (~50M). * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ */ ! public interface IMetadataService { /** * The approximate number of entries in the index (non-transactional). */ ! public int getEntryCount(String name); /** --- 56,75 ---- * <p> * The metadata service maintains locator information for the data service ! * instances responsible for each partition in the named index. Partitions are ! * automatically split when they overflow (~200M) and joined when they underflow ! * (~50M). ! * <p> ! * Note: methods on this interface MUST throw {@link IOException} in order to be ! * compatible with RMI. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ */ ! public interface IMetadataService extends Remote { /** * The approximate number of entries in the index (non-transactional). */ ! public int getEntryCount(String name) throws IOException; /** *************** *** 76,80 **** * @return */ ! public int rangeCount(String name,byte[] fromKey,byte[] toKey); /** --- 81,85 ---- * @return */ ! public int rangeCount(String name,byte[] fromKey,byte[] toKey) throws IOException; /** *************** *** 95,99 **** * index partitions surrounding that partition. */ ! public InetSocketAddress getDataService(String name, byte[] key); } --- 100,104 ---- * index partitions surrounding that partition. */ ! public InetSocketAddress getDataService(String name, byte[] key) throws IOException; } |
