Menu
▾
▴
[Archive-access-cvs] SF.net SVN: archive-access: [2425] trunk/archive-access/projects/wayback/ wayback-core/src/main/java/org/archive/wayback
|
From: <bra...@us...> - 2008-07-10 18:41:47
|
Revision: 2425
http://archive-access.svn.sourceforge.net/archive-access/?rev=2425&view=rev
Author: bradtofel
Date: 2008-07-10 11:41:53 -0700 (Thu, 10 Jul 2008)
Log Message:
-----------
REFACTOR: moved org.archive.wayback.bdb.* to org.archive.wayback.util.bdb.*
Modified Paths:
--------------
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ui/AdministrativeExclusionAuthority.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/bdb/BDBIndex.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/bdb/BDBRecordToSearchResultAdapter.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/bdb/SearchResultToBDBRecordAdapter.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/BDBResourceFileLocationDB.java
Added Paths:
-----------
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/util/bdb/
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/util/bdb/BDBRecord.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/util/bdb/BDBRecordIterator.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/util/bdb/BDBRecordSet.java
Removed Paths:
-------------
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/bdb/BDBRecord.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/bdb/BDBRecordIterator.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/bdb/BDBRecordSet.java
Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ui/AdministrativeExclusionAuthority.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ui/AdministrativeExclusionAuthority.java 2008-07-09 00:30:15 UTC (rev 2424)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ui/AdministrativeExclusionAuthority.java 2008-07-10 18:41:53 UTC (rev 2425)
@@ -27,10 +27,10 @@
import java.util.ArrayList;
import org.apache.commons.httpclient.URIException;
-import org.archive.wayback.bdb.BDBRecord;
-import org.archive.wayback.bdb.BDBRecordSet;
-import org.archive.wayback.bdb.BDBRecordIterator;
import org.archive.wayback.surt.SURTTokenizer;
+import org.archive.wayback.util.bdb.BDBRecord;
+import org.archive.wayback.util.bdb.BDBRecordIterator;
+import org.archive.wayback.util.bdb.BDBRecordSet;
import com.sleepycat.je.DatabaseException;
Deleted: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/bdb/BDBRecord.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/bdb/BDBRecord.java 2008-07-09 00:30:15 UTC (rev 2424)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/bdb/BDBRecord.java 2008-07-10 18:41:53 UTC (rev 2425)
@@ -1,58 +0,0 @@
-/* BDBRecord
- *
- * $Id$
- *
- * Created on 1:43:15 PM May 15, 2006.
- *
- * Copyright (C) 2006 Internet Archive.
- *
- * This file is part of wayback.
- *
- * wayback is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Lesser Public License as published by
- * the Free Software Foundation; either version 2.1 of the License, or
- * any later version.
- *
- * wayback is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Lesser Public License for more details.
- *
- * You should have received a copy of the GNU Lesser Public License
- * along with wayback; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
- */
-package org.archive.wayback.bdb;
-
-import com.sleepycat.je.DatabaseEntry;
-
-/**
- *
- *
- * @author brad
- * @version $Date$, $Revision$
- */
-public class BDBRecord {
- private DatabaseEntry key;
- private DatabaseEntry value;
- /**
- * @param key
- * @param value
- */
- public BDBRecord(DatabaseEntry key, DatabaseEntry value) {
- this.key = key;
- this.value = value;
- }
- /**
- * @return Returns the key.
- */
- public DatabaseEntry getKey() {
- return key;
- }
- /**
- * @return Returns the value.
- */
- public DatabaseEntry getValue() {
- return value;
- }
-}
Deleted: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/bdb/BDBRecordIterator.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/bdb/BDBRecordIterator.java 2008-07-09 00:30:15 UTC (rev 2424)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/bdb/BDBRecordIterator.java 2008-07-10 18:41:53 UTC (rev 2425)
@@ -1,155 +0,0 @@
-/* BDBRecordIterator
- *
- * $Id$
- *
- * Created on 1:27:24 PM May 15, 2006.
- *
- * Copyright (C) 2006 Internet Archive.
- *
- * This file is part of wayback.
- *
- * wayback is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Lesser Public License as published by
- * the Free Software Foundation; either version 2.1 of the License, or
- * any later version.
- *
- * wayback is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Lesser Public License for more details.
- *
- * You should have received a copy of the GNU Lesser Public License
- * along with wayback; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
- */
-package org.archive.wayback.bdb;
-
-import java.util.NoSuchElementException;
-
-import org.archive.wayback.util.CloseableIterator;
-
-import com.sleepycat.je.Cursor;
-import com.sleepycat.je.DatabaseEntry;
-import com.sleepycat.je.DatabaseException;
-import com.sleepycat.je.LockMode;
-import com.sleepycat.je.OperationStatus;
-
-/**
- *
- *
- * @author brad
- * @version $Date$, $Revision$
- */
-public class BDBRecordIterator implements CloseableIterator<BDBRecord> {
-
- DatabaseEntry key;
- DatabaseEntry value;
- boolean hitLast;
- boolean gotNext;
- Cursor cursor;
- boolean backward;
- BDBRecord record;
-
- /**
- * @param cursor
- * @param search
- * @throws DatabaseException
- */
- public BDBRecordIterator(Cursor cursor,String search)
- throws DatabaseException {
- initialize(cursor,search,false);
- }
- /**
- * @param cursor
- * @param search
- * @param backward
- * @throws DatabaseException
- */
- public BDBRecordIterator(Cursor cursor,String search,boolean backward)
- throws DatabaseException {
- initialize(cursor,search,backward);
- }
- private void initialize(Cursor cursor,String search, boolean backward)
- throws DatabaseException {
- this.cursor = cursor;
- this.backward = backward;
- key = new DatabaseEntry();
- value = new DatabaseEntry();
- key.setData(search.getBytes());
- key.setPartial(false);
- OperationStatus status = cursor.getSearchKeyRange(key, value,
- LockMode.DEFAULT);
- if(backward && (status == OperationStatus.SUCCESS)) {
- // if we are in reverse, immediately back up one record:
- status = cursor.getPrev(key, value, LockMode.DEFAULT);
- }
- if(status == OperationStatus.SUCCESS) {
- gotNext = true;
- }
- record = new BDBRecord(key,value);
- }
-
- /* (non-Javadoc)
- * @see java.util.Iterator#hasNext()
- */
- public boolean hasNext() {
- if(hitLast) {
- return false;
- }
- if(cursor == null) {
- return false;
- }
- if(!gotNext) {
- OperationStatus status;
- // attempt to get the next:
- try {
- if(backward) {
- status = cursor.getPrev(key, value, LockMode.DEFAULT);
- } else {
- status = cursor.getNext(key, value, LockMode.DEFAULT);
- }
- if(status == OperationStatus.SUCCESS) {
- gotNext = true;
- } else {
- close();
- }
- } catch (DatabaseException e) {
- // SLOP: throw a runtime?
- e.printStackTrace();
- close();
- }
- }
- return gotNext;
- }
-
- public void close() {
- if(!hitLast) {
- hitLast = true;
- try {
- cursor.close();
- } catch (DatabaseException e) {
- // TODO what to do?
- // let's just eat it for now..
- e.printStackTrace();
- }
- }
- }
-
- /* (non-Javadoc)
- * @see java.util.Iterator#next()
- */
- public BDBRecord next() {
- if(!gotNext) {
- throw new NoSuchElementException();
- }
- gotNext = false;
- return record;
- }
-
- /* (non-Javadoc)
- * @see java.util.Iterator#remove()
- */
- public void remove() {
- throw new UnsupportedOperationException();
- }
-}
Deleted: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/bdb/BDBRecordSet.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/bdb/BDBRecordSet.java 2008-07-09 00:30:15 UTC (rev 2424)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/bdb/BDBRecordSet.java 2008-07-10 18:41:53 UTC (rev 2425)
@@ -1,263 +0,0 @@
-/* BDBRecordGenerator
- *
- * $Id$
- *
- * Created on 1:22:39 PM May 15, 2006.
- *
- * Copyright (C) 2006 Internet Archive.
- *
- * This file is part of wayback.
- *
- * wayback is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Lesser Public License as published by
- * the Free Software Foundation; either version 2.1 of the License, or
- * any later version.
- *
- * wayback is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Lesser Public License for more details.
- *
- * You should have received a copy of the GNU Lesser Public License
- * along with wayback; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
- */
-package org.archive.wayback.bdb;
-
-import java.io.File;
-import java.io.UnsupportedEncodingException;
-import java.util.Iterator;
-
-import com.sleepycat.je.Cursor;
-import com.sleepycat.je.Database;
-import com.sleepycat.je.DatabaseConfig;
-import com.sleepycat.je.DatabaseEntry;
-import com.sleepycat.je.DatabaseException;
-import com.sleepycat.je.Environment;
-import com.sleepycat.je.EnvironmentConfig;
-import com.sleepycat.je.LockMode;
-import com.sleepycat.je.OperationStatus;
-import com.sleepycat.je.Transaction;
-
-/**
- *
- *
- * @author brad
- * @version $Date$, $Revision$
- */
-public class BDBRecordSet {
- /**
- * Maximum BDBJE file size
- */
- private final static String JE_LOG_FILEMAX = "256000000";
- /**
- * path to directory containing the BDBJE files
- */
- private String path;
-
- /**
- * name of BDBJE db within the path directory
- */
- private String dbName;
-
- /**
- * BDBJE Environment
- */
- Environment env = null;
-
- /**
- * BDBJE Database
- */
- Database db = null;
-
- /**
- * @param thePath Directory where BDBJE files are stored
- * @param theDbName Name of files in thePath
- * @throws DatabaseException
- */
- public void initializeDB(final String thePath, final String theDbName)
- throws DatabaseException {
- path = thePath;
- dbName = theDbName;
-
- EnvironmentConfig environmentConfig = new EnvironmentConfig();
- environmentConfig.setAllowCreate(true);
- environmentConfig.setTransactional(true);
- environmentConfig.setConfigParam("je.log.fileMax",JE_LOG_FILEMAX);
- File file = new File(path);
- if(!file.isDirectory()) {
- if(!file.mkdirs()) {
- throw new DatabaseException("failed mkdirs(" + path + ")");
- }
- }
- env = new Environment(file, environmentConfig);
- DatabaseConfig databaseConfig = new DatabaseConfig();
- databaseConfig.setAllowCreate(true);
- databaseConfig.setTransactional(true);
- // perform other database configurations
-
- db = env.openDatabase(null, dbName, databaseConfig);
- }
-
- /**
- * shut down the BDB.
- *
- * @throws DatabaseException
- */
- public synchronized void shutdownDB() throws DatabaseException {
-
- if (db != null) {
- db.close();
- db = null;
- }
-
- if (env != null) {
- env.close();
- env = null;
- }
- }
-
- /**
- * @param s
- * @return byte array representation of String s in UTF-8
- */
- public static byte[] stringToBytes(String s) {
- try {
- return s.getBytes("UTF-8");
- } catch (UnsupportedEncodingException e) {
- // no UTF-8, huh?
- e.printStackTrace();
- return s.getBytes();
- }
- }
- /**
- * @param ba
- * @return String of UTF-8 encoded bytes ba
- */
- public static String bytesToString(byte[] ba) {
- try {
- return new String(ba,"UTF-8");
- } catch (UnsupportedEncodingException e) {
- // not likely..
- e.printStackTrace();
- return new String(ba);
- }
- }
-
- /**
- * @param startKey
- * @return iterator for BDBRecords
- * @throws DatabaseException
- */
- public BDBRecordIterator recordIterator(final String startKey)
- throws DatabaseException {
- return recordIterator(startKey,true);
- }
- /**
- * @param startKey
- * @param forward
- * @return iterator for BDBRecords
- * @throws DatabaseException
- */
- public BDBRecordIterator recordIterator(final String startKey,
- final boolean forward) throws DatabaseException {
- Cursor cursor = db.openCursor(null, null);
- return new BDBRecordIterator(cursor,startKey,!forward);
- }
-
- /**
- * @param itr
- */
- public void insertRecords(final Iterator<BDBRecord> itr) {
- OperationStatus status = null;
- try {
- Transaction txn = env.beginTransaction(null, null);
- try {
- Cursor cursor = db.openCursor(txn, null);
- while (itr.hasNext()) {
- BDBRecord record = (BDBRecord) itr.next();
- status = cursor.put(record.getKey(), record.getValue());
- if (status != OperationStatus.SUCCESS) {
- throw new RuntimeException("put() non-success status");
- }
- }
- cursor.close();
- txn.commit();
- } catch (DatabaseException e) {
- if(txn != null) {
- txn.abort();
- }
- e.printStackTrace();
- }
- } catch (DatabaseException e) {
- e.printStackTrace();
- }
- }
-
- /**
- * persistantly store key-value pair
- * @param keyStr
- * @param valueStr
- * @throws DatabaseException
- */
- public void put(String keyStr, String valueStr) throws DatabaseException {
- DatabaseEntry key = new DatabaseEntry(stringToBytes(keyStr));
- DatabaseEntry data = new DatabaseEntry(stringToBytes(valueStr));
- db.put(null, key, data);
- }
-
- /**
- * retrieve the value assoicated with keyStr from persistant storage
- * @param keyStr
- * @return String value associated with key, or null if no key is found
- * or an error occurs
- * @throws DatabaseException
- */
- public String get(String keyStr) throws DatabaseException {
- String result = null;
- DatabaseEntry key = new DatabaseEntry(stringToBytes(keyStr));
- DatabaseEntry data = new DatabaseEntry();
- if (db.get(null, key, data, LockMode.DEFAULT)
- == OperationStatus.SUCCESS) {
-
- result = bytesToString(data.getData());
- }
- return result;
- }
-
- /**
- * @param keyStr
- * @throws DatabaseException
- */
- public void delete(String keyStr) throws DatabaseException {
- db.delete(null,new DatabaseEntry(stringToBytes(keyStr)));
- }
-
- /**
- * @return Returns the dbName.
- */
- public String getDbName() {
- return dbName;
- }
-
- /**
- * @return Returns the path.
- */
- public String getPath() {
- return path;
- }
-
- /**
- * @param path the path to set
- */
- public void setPath(String path) {
- this.path = path;
- }
-
- /**
- * @param dbName the dbName to set
- */
- public void setDbName(String dbName) {
- this.dbName = dbName;
- }
-}
Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/bdb/BDBIndex.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/bdb/BDBIndex.java 2008-07-09 00:30:15 UTC (rev 2424)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/bdb/BDBIndex.java 2008-07-10 18:41:53 UTC (rev 2425)
@@ -31,8 +31,6 @@
import java.util.Iterator;
import org.archive.wayback.UrlCanonicalizer;
-import org.archive.wayback.bdb.BDBRecord;
-import org.archive.wayback.bdb.BDBRecordSet;
import org.archive.wayback.core.CaptureSearchResult;
import org.archive.wayback.exception.ConfigurationException;
import org.archive.wayback.exception.ResourceIndexNotAvailableException;
@@ -42,6 +40,8 @@
import org.archive.wayback.util.AdaptedIterator;
import org.archive.wayback.util.Adapter;
import org.archive.wayback.util.CloseableIterator;
+import org.archive.wayback.util.bdb.BDBRecord;
+import org.archive.wayback.util.bdb.BDBRecordSet;
import org.archive.wayback.util.flatfile.RecordIterator;
import org.archive.wayback.util.url.AggressiveUrlCanonicalizer;
Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/bdb/BDBRecordToSearchResultAdapter.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/bdb/BDBRecordToSearchResultAdapter.java 2008-07-09 00:30:15 UTC (rev 2424)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/bdb/BDBRecordToSearchResultAdapter.java 2008-07-10 18:41:53 UTC (rev 2425)
@@ -26,10 +26,10 @@
import java.io.UnsupportedEncodingException;
-import org.archive.wayback.bdb.BDBRecord;
import org.archive.wayback.core.CaptureSearchResult;
import org.archive.wayback.resourceindex.cdx.CDXLineToSearchResultAdapter;
import org.archive.wayback.util.Adapter;
+import org.archive.wayback.util.bdb.BDBRecord;
/**
* Adapter that converts a BDBRecord into a SearchResult
Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/bdb/SearchResultToBDBRecordAdapter.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/bdb/SearchResultToBDBRecordAdapter.java 2008-07-09 00:30:15 UTC (rev 2424)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/bdb/SearchResultToBDBRecordAdapter.java 2008-07-10 18:41:53 UTC (rev 2425)
@@ -28,10 +28,10 @@
import org.apache.commons.httpclient.URIException;
import org.archive.wayback.UrlCanonicalizer;
-import org.archive.wayback.bdb.BDBRecord;
-import org.archive.wayback.bdb.BDBRecordSet;
import org.archive.wayback.core.CaptureSearchResult;
import org.archive.wayback.util.Adapter;
+import org.archive.wayback.util.bdb.BDBRecord;
+import org.archive.wayback.util.bdb.BDBRecordSet;
import com.sleepycat.je.DatabaseEntry;
Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/BDBResourceFileLocationDB.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/BDBResourceFileLocationDB.java 2008-07-09 00:30:15 UTC (rev 2424)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/BDBResourceFileLocationDB.java 2008-07-10 18:41:53 UTC (rev 2425)
@@ -28,10 +28,10 @@
import java.io.IOException;
import java.io.InputStreamReader;
-import org.archive.wayback.bdb.BDBRecordSet;
import org.archive.wayback.resourcestore.locationdb.ResourceFileLocationDB;
import org.archive.wayback.resourcestore.locationdb.ResourceFileLocationDBLog;
import org.archive.wayback.util.CloseableIterator;
+import org.archive.wayback.util.bdb.BDBRecordSet;
import com.sleepycat.je.DatabaseException;
Copied: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/util/bdb/BDBRecord.java (from rev 2418, trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/bdb/BDBRecord.java)
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/util/bdb/BDBRecord.java (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/util/bdb/BDBRecord.java 2008-07-10 18:41:53 UTC (rev 2425)
@@ -0,0 +1,58 @@
+/* BDBRecord
+ *
+ * $Id$
+ *
+ * Created on 1:43:15 PM May 15, 2006.
+ *
+ * Copyright (C) 2006 Internet Archive.
+ *
+ * This file is part of wayback.
+ *
+ * wayback is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * any later version.
+ *
+ * wayback is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser Public License
+ * along with wayback; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+package org.archive.wayback.util.bdb;
+
+import com.sleepycat.je.DatabaseEntry;
+
+/**
+ *
+ *
+ * @author brad
+ * @version $Date$, $Revision$
+ */
+public class BDBRecord {
+ private DatabaseEntry key;
+ private DatabaseEntry value;
+ /**
+ * @param key
+ * @param value
+ */
+ public BDBRecord(DatabaseEntry key, DatabaseEntry value) {
+ this.key = key;
+ this.value = value;
+ }
+ /**
+ * @return Returns the key.
+ */
+ public DatabaseEntry getKey() {
+ return key;
+ }
+ /**
+ * @return Returns the value.
+ */
+ public DatabaseEntry getValue() {
+ return value;
+ }
+}
Copied: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/util/bdb/BDBRecordIterator.java (from rev 2418, trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/bdb/BDBRecordIterator.java)
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/util/bdb/BDBRecordIterator.java (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/util/bdb/BDBRecordIterator.java 2008-07-10 18:41:53 UTC (rev 2425)
@@ -0,0 +1,155 @@
+/* BDBRecordIterator
+ *
+ * $Id$
+ *
+ * Created on 1:27:24 PM May 15, 2006.
+ *
+ * Copyright (C) 2006 Internet Archive.
+ *
+ * This file is part of wayback.
+ *
+ * wayback is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * any later version.
+ *
+ * wayback is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser Public License
+ * along with wayback; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+package org.archive.wayback.util.bdb;
+
+import java.util.NoSuchElementException;
+
+import org.archive.wayback.util.CloseableIterator;
+
+import com.sleepycat.je.Cursor;
+import com.sleepycat.je.DatabaseEntry;
+import com.sleepycat.je.DatabaseException;
+import com.sleepycat.je.LockMode;
+import com.sleepycat.je.OperationStatus;
+
+/**
+ *
+ *
+ * @author brad
+ * @version $Date$, $Revision$
+ */
+public class BDBRecordIterator implements CloseableIterator<BDBRecord> {
+
+ DatabaseEntry key;
+ DatabaseEntry value;
+ boolean hitLast;
+ boolean gotNext;
+ Cursor cursor;
+ boolean backward;
+ BDBRecord record;
+
+ /**
+ * @param cursor
+ * @param search
+ * @throws DatabaseException
+ */
+ public BDBRecordIterator(Cursor cursor,String search)
+ throws DatabaseException {
+ initialize(cursor,search,false);
+ }
+ /**
+ * @param cursor
+ * @param search
+ * @param backward
+ * @throws DatabaseException
+ */
+ public BDBRecordIterator(Cursor cursor,String search,boolean backward)
+ throws DatabaseException {
+ initialize(cursor,search,backward);
+ }
+ private void initialize(Cursor cursor,String search, boolean backward)
+ throws DatabaseException {
+ this.cursor = cursor;
+ this.backward = backward;
+ key = new DatabaseEntry();
+ value = new DatabaseEntry();
+ key.setData(search.getBytes());
+ key.setPartial(false);
+ OperationStatus status = cursor.getSearchKeyRange(key, value,
+ LockMode.DEFAULT);
+ if(backward && (status == OperationStatus.SUCCESS)) {
+ // if we are in reverse, immediately back up one record:
+ status = cursor.getPrev(key, value, LockMode.DEFAULT);
+ }
+ if(status == OperationStatus.SUCCESS) {
+ gotNext = true;
+ }
+ record = new BDBRecord(key,value);
+ }
+
+ /* (non-Javadoc)
+ * @see java.util.Iterator#hasNext()
+ */
+ public boolean hasNext() {
+ if(hitLast) {
+ return false;
+ }
+ if(cursor == null) {
+ return false;
+ }
+ if(!gotNext) {
+ OperationStatus status;
+ // attempt to get the next:
+ try {
+ if(backward) {
+ status = cursor.getPrev(key, value, LockMode.DEFAULT);
+ } else {
+ status = cursor.getNext(key, value, LockMode.DEFAULT);
+ }
+ if(status == OperationStatus.SUCCESS) {
+ gotNext = true;
+ } else {
+ close();
+ }
+ } catch (DatabaseException e) {
+ // SLOP: throw a runtime?
+ e.printStackTrace();
+ close();
+ }
+ }
+ return gotNext;
+ }
+
+ public void close() {
+ if(!hitLast) {
+ hitLast = true;
+ try {
+ cursor.close();
+ } catch (DatabaseException e) {
+ // TODO what to do?
+ // let's just eat it for now..
+ e.printStackTrace();
+ }
+ }
+ }
+
+ /* (non-Javadoc)
+ * @see java.util.Iterator#next()
+ */
+ public BDBRecord next() {
+ if(!gotNext) {
+ throw new NoSuchElementException();
+ }
+ gotNext = false;
+ return record;
+ }
+
+ /* (non-Javadoc)
+ * @see java.util.Iterator#remove()
+ */
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+}
Copied: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/util/bdb/BDBRecordSet.java (from rev 2418, trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/bdb/BDBRecordSet.java)
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/util/bdb/BDBRecordSet.java (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/util/bdb/BDBRecordSet.java 2008-07-10 18:41:53 UTC (rev 2425)
@@ -0,0 +1,264 @@
+/* BDBRecordGenerator
+ *
+ * $Id$
+ *
+ * Created on 1:22:39 PM May 15, 2006.
+ *
+ * Copyright (C) 2006 Internet Archive.
+ *
+ * This file is part of wayback.
+ *
+ * wayback is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * any later version.
+ *
+ * wayback is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser Public License
+ * along with wayback; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+package org.archive.wayback.util.bdb;
+
+import java.io.File;
+import java.io.UnsupportedEncodingException;
+import java.util.Iterator;
+
+
+import com.sleepycat.je.Cursor;
+import com.sleepycat.je.Database;
+import com.sleepycat.je.DatabaseConfig;
+import com.sleepycat.je.DatabaseEntry;
+import com.sleepycat.je.DatabaseException;
+import com.sleepycat.je.Environment;
+import com.sleepycat.je.EnvironmentConfig;
+import com.sleepycat.je.LockMode;
+import com.sleepycat.je.OperationStatus;
+import com.sleepycat.je.Transaction;
+
+/**
+ *
+ *
+ * @author brad
+ * @version $Date$, $Revision$
+ */
+public class BDBRecordSet {
+ /**
+ * Maximum BDBJE file size
+ */
+ private final static String JE_LOG_FILEMAX = "256000000";
+ /**
+ * path to directory containing the BDBJE files
+ */
+ private String path;
+
+ /**
+ * name of BDBJE db within the path directory
+ */
+ private String dbName;
+
+ /**
+ * BDBJE Environment
+ */
+ Environment env = null;
+
+ /**
+ * BDBJE Database
+ */
+ Database db = null;
+
+ /**
+ * @param thePath Directory where BDBJE files are stored
+ * @param theDbName Name of files in thePath
+ * @throws DatabaseException
+ */
+ public void initializeDB(final String thePath, final String theDbName)
+ throws DatabaseException {
+ path = thePath;
+ dbName = theDbName;
+
+ EnvironmentConfig environmentConfig = new EnvironmentConfig();
+ environmentConfig.setAllowCreate(true);
+ environmentConfig.setTransactional(true);
+ environmentConfig.setConfigParam("je.log.fileMax",JE_LOG_FILEMAX);
+ File file = new File(path);
+ if(!file.isDirectory()) {
+ if(!file.mkdirs()) {
+ throw new DatabaseException("failed mkdirs(" + path + ")");
+ }
+ }
+ env = new Environment(file, environmentConfig);
+ DatabaseConfig databaseConfig = new DatabaseConfig();
+ databaseConfig.setAllowCreate(true);
+ databaseConfig.setTransactional(true);
+ // perform other database configurations
+
+ db = env.openDatabase(null, dbName, databaseConfig);
+ }
+
+ /**
+ * shut down the BDB.
+ *
+ * @throws DatabaseException
+ */
+ public synchronized void shutdownDB() throws DatabaseException {
+
+ if (db != null) {
+ db.close();
+ db = null;
+ }
+
+ if (env != null) {
+ env.close();
+ env = null;
+ }
+ }
+
+ /**
+ * @param s
+ * @return byte array representation of String s in UTF-8
+ */
+ public static byte[] stringToBytes(String s) {
+ try {
+ return s.getBytes("UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ // no UTF-8, huh?
+ e.printStackTrace();
+ return s.getBytes();
+ }
+ }
+ /**
+ * @param ba
+ * @return String of UTF-8 encoded bytes ba
+ */
+ public static String bytesToString(byte[] ba) {
+ try {
+ return new String(ba,"UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ // not likely..
+ e.printStackTrace();
+ return new String(ba);
+ }
+ }
+
+ /**
+ * @param startKey
+ * @return iterator for BDBRecords
+ * @throws DatabaseException
+ */
+ public BDBRecordIterator recordIterator(final String startKey)
+ throws DatabaseException {
+ return recordIterator(startKey,true);
+ }
+ /**
+ * @param startKey
+ * @param forward
+ * @return iterator for BDBRecords
+ * @throws DatabaseException
+ */
+ public BDBRecordIterator recordIterator(final String startKey,
+ final boolean forward) throws DatabaseException {
+ Cursor cursor = db.openCursor(null, null);
+ return new BDBRecordIterator(cursor,startKey,!forward);
+ }
+
+ /**
+ * @param itr
+ */
+ public void insertRecords(final Iterator<BDBRecord> itr) {
+ OperationStatus status = null;
+ try {
+ Transaction txn = env.beginTransaction(null, null);
+ try {
+ Cursor cursor = db.openCursor(txn, null);
+ while (itr.hasNext()) {
+ BDBRecord record = (BDBRecord) itr.next();
+ status = cursor.put(record.getKey(), record.getValue());
+ if (status != OperationStatus.SUCCESS) {
+ throw new RuntimeException("put() non-success status");
+ }
+ }
+ cursor.close();
+ txn.commit();
+ } catch (DatabaseException e) {
+ if(txn != null) {
+ txn.abort();
+ }
+ e.printStackTrace();
+ }
+ } catch (DatabaseException e) {
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * persistantly store key-value pair
+ * @param keyStr
+ * @param valueStr
+ * @throws DatabaseException
+ */
+ public void put(String keyStr, String valueStr) throws DatabaseException {
+ DatabaseEntry key = new DatabaseEntry(stringToBytes(keyStr));
+ DatabaseEntry data = new DatabaseEntry(stringToBytes(valueStr));
+ db.put(null, key, data);
+ }
+
+ /**
+ * retrieve the value assoicated with keyStr from persistant storage
+ * @param keyStr
+ * @return String value associated with key, or null if no key is found
+ * or an error occurs
+ * @throws DatabaseException
+ */
+ public String get(String keyStr) throws DatabaseException {
+ String result = null;
+ DatabaseEntry key = new DatabaseEntry(stringToBytes(keyStr));
+ DatabaseEntry data = new DatabaseEntry();
+ if (db.get(null, key, data, LockMode.DEFAULT)
+ == OperationStatus.SUCCESS) {
+
+ result = bytesToString(data.getData());
+ }
+ return result;
+ }
+
+ /**
+ * @param keyStr
+ * @throws DatabaseException
+ */
+ public void delete(String keyStr) throws DatabaseException {
+ db.delete(null,new DatabaseEntry(stringToBytes(keyStr)));
+ }
+
+ /**
+ * @return Returns the dbName.
+ */
+ public String getDbName() {
+ return dbName;
+ }
+
+ /**
+ * @return Returns the path.
+ */
+ public String getPath() {
+ return path;
+ }
+
+ /**
+ * @param path the path to set
+ */
+ public void setPath(String path) {
+ this.path = path;
+ }
+
+ /**
+ * @param dbName the dbName to set
+ */
+ public void setDbName(String dbName) {
+ this.dbName = dbName;
+ }
+}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
