Menu
▾
▴
[Archive-access-cvs] SF.net SVN: archive-access: [2286] trunk/archive-access/projects/wayback/ wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb
|
From: <bra...@us...> - 2008-06-05 21:52:47
|
Revision: 2286
http://archive-access.svn.sourceforge.net/archive-access/?rev=2286&view=rev
Author: bradtofel
Date: 2008-06-05 14:52:54 -0700 (Thu, 05 Jun 2008)
Log Message:
-----------
INTERFACE: changed ResourceFileLocationDB to an interface, with method not specific to ARCs, created BDB and remote implementations of that interface.
Modified Paths:
--------------
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileProxyServlet.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/ResourceFileLocationDBLog.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/ResourceFileLocationDBServlet.java
Added Paths:
-----------
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/BDBResourceFileLocationDB.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/RemoteResourceFileLocationDB.java
Removed Paths:
-------------
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/ResourceFileLocationDB.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/ResourceFileLocationDBClient.java
Copied: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/BDBResourceFileLocationDB.java (from rev 2284, trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/ResourceFileLocationDB.java)
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/BDBResourceFileLocationDB.java (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/BDBResourceFileLocationDB.java 2008-06-05 21:52:54 UTC (rev 2286)
@@ -0,0 +1,316 @@
+/* BDBResourceFileLocationDB
+ *
+ * $Id$
+ *
+ * Created on 3:08:59 PM Aug 18, 2006.
+ *
+ * Copyright (C) 2006 Internet Archive.
+ *
+ * This file is part of Wayback.
+ *
+ * Wayback is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * any later version.
+ *
+ * Wayback is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser Public License
+ * along with Wayback; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+package org.archive.wayback.resourcestore.locationdb;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+import org.archive.wayback.bdb.BDBRecordSet;
+import org.archive.wayback.resourcestore.locationdb2.ResourceFileLocationDB;
+import org.archive.wayback.resourcestore.locationdb2.ResourceFileLocationDBLog;
+import org.archive.wayback.util.CloseableIterator;
+
+import com.sleepycat.je.DatabaseException;
+
+/**
+ *
+ *
+ * @author brad
+ * @version $Date$, $Revision$
+ */
+public class BDBResourceFileLocationDB implements ResourceFileLocationDB {
+
+ private final static String urlDelimiter = " ";
+
+ private final static String urlDelimiterRE = " ";
+
+ private ResourceFileLocationDBLog log;
+ private BDBRecordSet bdb = null;
+ private String logPath = null;
+ private String bdbPath = null;
+ private String bdbName = null;
+
+ private IOException wrapDBException(DatabaseException e) {
+ return new IOException(e.getLocalizedMessage());
+ }
+ private String get(String key) throws IOException {
+ try {
+ return bdb.get(key);
+ } catch (DatabaseException e) {
+ throw wrapDBException(e);
+ }
+ }
+ private void put(String key, String value) throws IOException {
+ try {
+ bdb.put(key,value);
+ } catch (DatabaseException e) {
+ throw wrapDBException(e);
+ }
+ }
+ private void delete(String key) throws IOException {
+ try {
+ bdb.delete(key);
+ } catch (DatabaseException e) {
+ throw wrapDBException(e);
+ }
+ }
+ public void shutdown() throws IOException {
+ try {
+ bdb.shutdownDB();
+ } catch (DatabaseException e) {
+ throw wrapDBException(e);
+ }
+ }
+
+ public void init() throws IOException {
+ if(logPath == null) {
+ throw new IOException("No logPath");
+ }
+ log = new ResourceFileLocationDBLog(logPath);
+ bdb = new BDBRecordSet();
+ try {
+ bdb.initializeDB(bdbPath,bdbName);
+ } catch (DatabaseException e) {
+ throw wrapDBException(e);
+ }
+ }
+
+ /**
+ * return an array of String URLs for all known locations of name in the DB.
+ * @param name
+ * @return String[] of URLs to name
+ * @throws IOException
+ */
+ public String[] nameToUrls(final String name) throws IOException {
+
+ String[] urls = null;
+ String valueString = get(name);
+ if(valueString != null && valueString.length() > 0) {
+ urls = valueString.split(urlDelimiterRE);
+ }
+ return urls;
+ }
+
+ /**
+ * add an url location for a name, unless it already exists
+ * @param name
+ * @param url
+ * @throws IOException
+ */
+ public void addNameUrl(final String name, final String url)
+ throws IOException {
+
+ // need to first see if there is already an entry for this name.
+ // if not, add url as the value.
+ // if so, check the current url locations for name
+ // if url exists, do nothing
+ // if url does not exist, add, and set that as the value.
+
+ String newValue = null;
+ String oldValue = get(name);
+ if(oldValue != null && oldValue.length() > 0) {
+ String curUrls[] = oldValue.split(urlDelimiterRE);
+ boolean found = false;
+ for(int i=0; i < curUrls.length; i++) {
+ if(url.equals(curUrls[i])) {
+ found = true;
+ break;
+ }
+ }
+ if(found == false) {
+ newValue = oldValue + " " + url;
+ }
+ } else {
+ // null or empty value
+ newValue = url;
+ if(oldValue == null) log.addFile(name);
+ }
+
+ // did we find a value?
+ if(newValue != null) {
+ put(name,newValue);
+ }
+ }
+
+ /**
+ * remove a single url location for an name, if it exists
+ * @param name
+ * @param url
+ * @throws IOException
+ */
+ public void removeNameUrl(final String name, final String url)
+ throws IOException {
+ // need to first see if there is already an entry for this name.
+ // if not, do nothing
+ // if so, loop thru all current url locations for name
+ // keep any that are not url
+ // if any locations are left, update to the new value, sans url
+ // if none are left, remove the entry from the db
+
+ StringBuilder newValue = new StringBuilder();
+ String oldValue = get(name);
+ if(oldValue != null && oldValue.length() > 0) {
+ String curUrls[] = oldValue.split(urlDelimiterRE);
+
+ for(int i=0; i < curUrls.length; i++) {
+ if(!url.equals(curUrls[i])) {
+ if(newValue.length() > 0) {
+ newValue.append(urlDelimiter);
+ }
+ newValue.append(curUrls[i]);
+ }
+ }
+
+ if(newValue.length() > 0) {
+
+ // update
+ put(name, newValue.toString());
+
+ } else {
+
+ // remove the entry:
+ delete(name);
+ }
+ }
+ }
+
+ /**
+ * @param start
+ * @param end
+ * @return Iterator for traversing arcs between start and end.
+ * @throws IOException
+ */
+ public CloseableIterator<String> getNamesBetweenMarks(long start, long end)
+ throws IOException {
+ return log.getNamesBetweenMarks(start, end);
+ }
+
+ /**
+ * @return current "Mark" for the log. Currently, it's just the length of
+ * the log file.
+ */
+ public long getCurrentMark() {
+ return log.getCurrentMark();
+ }
+
+ /**
+ * @return the logPath
+ */
+ public String getLogPath() {
+ return logPath;
+ }
+
+ /**
+ * @param logPath the logPath to set
+ */
+ public void setLogPath(String logPath) {
+ this.logPath = logPath;
+ }
+
+ /**
+ * @return the bdbPath
+ */
+ public String getBdbPath() {
+ return bdbPath;
+ }
+
+ /**
+ * @param bdbPath the bdbPath to set
+ */
+ public void setBdbPath(String bdbPath) {
+ this.bdbPath = bdbPath;
+ }
+
+ /**
+ * @return the bdbName
+ */
+ public String getBdbName() {
+ return bdbName;
+ }
+
+ /**
+ * @param bdbName the bdbName to set
+ */
+ public void setBdbName(String bdbName) {
+ this.bdbName = bdbName;
+ }
+ private static void USAGE(String message) {
+ System.err.print("USAGE: " + message + "\n" +
+ "\tDBDIR DBNAME LOGPATH\n" +
+ "\n" +
+ "\t\tread lines from STDIN formatted like:\n" +
+ "\t\t\tNAME<SPACE>URL\n" +
+ "\t\tand for each line, add to locationDB that file NAME is\n" +
+ "\t\tlocated at URL. Use locationDB in DBDIR at DBNAME, \n" +
+ "\t\tcreating if it does not exist.\n"
+ );
+ System.exit(2);
+ }
+
+ /**
+ * @param args
+ */
+ public static void main(String[] args) {
+ if(args.length != 3) {
+ USAGE("");
+ System.exit(1);
+ }
+ String bdbPath = args[0];
+ String bdbName = args[1];
+ String logPath = args[2];
+ BDBResourceFileLocationDB db = new BDBResourceFileLocationDB();
+ db.setBdbPath(bdbPath);
+ db.setBdbName(bdbName);
+ db.setLogPath(logPath);
+ BufferedReader r = new BufferedReader(
+ new InputStreamReader(System.in));
+ String line;
+ int exitCode = 0;
+ try {
+ db.init();
+ while((line = r.readLine()) != null) {
+ String parts[] = line.split(" ");
+ if(parts.length != 2) {
+ System.err.println("Bad input(" + line + ")");
+ System.exit(2);
+ }
+ db.addNameUrl(parts[0],parts[1]);
+ System.out.println("Added\t" + parts[0] + "\t" + parts[1]);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ exitCode = 1;
+ } finally {
+ try {
+ db.shutdown();
+ } catch (IOException e) {
+ e.printStackTrace();
+ exitCode = 1;
+ }
+ }
+ System.exit(exitCode);
+ }
+}
Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileProxyServlet.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileProxyServlet.java 2008-06-05 21:37:36 UTC (rev 2285)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileProxyServlet.java 2008-06-05 21:52:54 UTC (rev 2286)
@@ -29,20 +29,19 @@
import java.io.OutputStream;
import java.net.URL;
import java.net.URLConnection;
-import java.text.ParseException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import org.archive.wayback.resourcestore.locationdb.ResourceFileLocationDB;
import org.archive.wayback.webapp.ServletRequestContext;
-import com.sleepycat.je.DatabaseException;
-
/**
+ * ServletRequestContext interface which uses a ResourceFileLocationDB to
+ * reverse proxy an incoming HTTP request for a file by name to it's actual
+ * back-end location. This will also forward HTTP byte range requests to the
+ * final location.
*
- *
* @author brad
* @version $Date$, $Revision$
*/
@@ -61,49 +60,46 @@
HttpServletResponse httpResponse) throws IOException,
ServletException {
- try {
- String arc = httpRequest.getRequestURI();
- arc = arc.substring(arc.lastIndexOf('/')+1);
- if(arc.length() == 0) {
- throw new ParseException("no/invalid arc",0);
- }
- String urls[] = locationDB.arcToUrls(arc);
+ String name = httpRequest.getRequestURI();
+ name = name.substring(name.lastIndexOf('/')+1);
+ if(name.length() == 0) {
+ httpResponse.sendError(HttpServletResponse.SC_BAD_REQUEST,
+ "no/invalid name");
+ } else {
+
+ String urls[] = locationDB.nameToUrls(name);
+
if(urls == null || urls.length == 0) {
- throw new DatabaseException("Unable to locate("+arc+")");
- }
- String urlString = urls[0];
- String rangeHeader = httpRequest.getHeader(RANGE_HTTP_HEADER);
- URL url = new URL(urlString);
- URLConnection conn = url.openConnection();
- if(rangeHeader != null) {
- conn.addRequestProperty(RANGE_HTTP_HEADER,rangeHeader);
- }
- InputStream is = conn.getInputStream();
- httpResponse.setStatus(HttpServletResponse.SC_OK);
- String typeHeader = conn.getHeaderField(CONTENT_TYPE_HEADER);
- if(typeHeader == null) {
- typeHeader = CONTENT_TYPE;
- }
- httpResponse.setContentType(typeHeader);
- OutputStream os = httpResponse.getOutputStream();
- int BUF_SIZE = 4096;
- byte[] buffer = new byte[BUF_SIZE];
- try {
- for (int r = -1; (r = is.read(buffer, 0, BUF_SIZE)) != -1;) {
- os.write(buffer, 0, r);
+
+ httpResponse.sendError(HttpServletResponse.SC_NOT_FOUND,
+ "Unable to locate("+name+")");
+ } else {
+
+ String urlString = urls[0];
+ String rangeHeader = httpRequest.getHeader(RANGE_HTTP_HEADER);
+ URL url = new URL(urlString);
+ URLConnection conn = url.openConnection();
+ if(rangeHeader != null) {
+ conn.addRequestProperty(RANGE_HTTP_HEADER,rangeHeader);
}
- } finally {
- is.close();
+ InputStream is = conn.getInputStream();
+ httpResponse.setStatus(HttpServletResponse.SC_OK);
+ String typeHeader = conn.getHeaderField(CONTENT_TYPE_HEADER);
+ if(typeHeader == null) {
+ typeHeader = CONTENT_TYPE;
+ }
+ httpResponse.setContentType(typeHeader);
+ OutputStream os = httpResponse.getOutputStream();
+ int BUF_SIZE = 4096;
+ byte[] buffer = new byte[BUF_SIZE];
+ try {
+ for(int r = -1; (r = is.read(buffer, 0, BUF_SIZE)) != -1;) {
+ os.write(buffer, 0, r);
+ }
+ } finally {
+ is.close();
+ }
}
- } catch (ParseException e) {
- e.printStackTrace();
- httpResponse.sendError(HttpServletResponse.SC_BAD_REQUEST,
- e.getMessage());
- } catch (DatabaseException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- httpResponse.sendError(HttpServletResponse.SC_NOT_FOUND,
- e.getMessage());
}
return true;
}
Copied: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/RemoteResourceFileLocationDB.java (from rev 2284, trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/ResourceFileLocationDBClient.java)
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/RemoteResourceFileLocationDB.java (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/RemoteResourceFileLocationDB.java 2008-06-05 21:52:54 UTC (rev 2286)
@@ -0,0 +1,427 @@
+/* FileLocationDBClient
+ *
+ * $Id$
+ *
+ * Created on 5:59:49 PM Aug 21, 2006.
+ *
+ * Copyright (C) 2006 Internet Archive.
+ *
+ * This file is part of Wayback.
+ *
+ * Wayback is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * any later version.
+ *
+ * Wayback is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser Public License
+ * along with Wayback; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+package org.archive.wayback.resourcestore.locationdb;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.logging.Logger;
+
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.HttpStatus;
+import org.apache.commons.httpclient.NameValuePair;
+import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.commons.httpclient.methods.PostMethod;
+import org.apache.commons.httpclient.util.ParameterFormatter;
+import org.archive.wayback.resourcestore.locationdb.ResourceFileLocationDBServlet;
+import org.archive.wayback.util.CloseableIterator;
+import org.archive.wayback.util.WrappedCloseableIterator;
+
+/**
+ *
+ *
+ * @author brad
+ * @version $Date$, $Revision$
+ */
+public class RemoteResourceFileLocationDB implements ResourceFileLocationDB {
+ private static final Logger LOGGER = Logger.getLogger(RemoteResourceFileLocationDB
+ .class.getName());
+
+ private final static String ARC_SUFFIX = ".arc";
+ private final static String ARC_GZ_SUFFIX = ".arc.gz";
+ private final static String WARC_SUFFIX = ".warc";
+ private final static String WARC_GZ_SUFFIX = ".warc.gz";
+ private final static String OK_RESPONSE_PREFIX = "OK ";
+ private HttpClient client = null;
+
+ private String serverUrl = null;
+
+ /**
+ * @param serverUrl
+ */
+ public RemoteResourceFileLocationDB(final String serverUrl) {
+ super();
+ this.serverUrl = serverUrl;
+ this.client = new HttpClient();
+ }
+
+ /**
+ * @return long value representing the current end "mark" of the db log
+ * @throws IOException
+ */
+ public long getCurrentMark() throws IOException {
+ NameValuePair[] args = {
+ new NameValuePair(
+ ResourceFileLocationDBServlet.OPERATION_ARGUMENT,
+ ResourceFileLocationDBServlet.GETMARK_OPERATION),
+ };
+ return Long.parseLong(doGetMethod(args));
+ }
+
+ /**
+ * @param start
+ * @param end
+ * @return Iterator of file names between marks start and end
+ * @throws IOException
+ */
+ public CloseableIterator<String> getNamesBetweenMarks(long start, long end)
+ throws IOException {
+ NameValuePair[] args = {
+ new NameValuePair(
+ ResourceFileLocationDBServlet.OPERATION_ARGUMENT,
+ ResourceFileLocationDBServlet.GETRANGE_OPERATION),
+ new NameValuePair(
+ ResourceFileLocationDBServlet.START_ARGUMENT,
+ String.valueOf(start)),
+ new NameValuePair(
+ ResourceFileLocationDBServlet.END_ARGUMENT,
+ String.valueOf(end))
+ };
+ return new WrappedCloseableIterator<String>(
+ Arrays.asList(doGetMethod(args).split("\n")).iterator());
+ }
+
+ /**
+ * return an array of String URLs for all known locations of the file
+ * in the DB.
+ * @param name
+ * @return String[] of URLs to arcName
+ * @throws IOException
+ */
+ public String[] nameToUrls(final String name) throws IOException {
+
+ NameValuePair[] args = {
+ new NameValuePair(
+ ResourceFileLocationDBServlet.OPERATION_ARGUMENT,
+ ResourceFileLocationDBServlet.LOOKUP_OPERATION),
+
+ new NameValuePair(
+ ResourceFileLocationDBServlet.NAME_ARGUMENT,
+ name)
+ };
+ String locations = doGetMethod(args);
+ if(locations != null) {
+ return locations.split("\n");
+ }
+ return null;
+ }
+
+
+ /**
+ * add an Url location for an arcName, unless it already exists
+ * @param name
+ * @param url
+ * @throws IOException
+ */
+ public void addNameUrl(final String name, final String url)
+ throws IOException {
+ doPostMethod(ResourceFileLocationDBServlet.ADD_OPERATION, name, url);
+ }
+
+ /**
+ * remove a single url location for a name, if it exists
+ * @param name
+ * @param url
+ * @throws IOException
+ */
+ public void removeNameUrl(final String name, final String url)
+ throws IOException {
+ doPostMethod(ResourceFileLocationDBServlet.REMOVE_OPERATION, name, url);
+ }
+
+ private String doGetMethod(NameValuePair[] data) throws IOException {
+ ParameterFormatter formatter = new ParameterFormatter();
+ formatter.setAlwaysUseQuotes(false);
+ StringBuilder finalUrl = new StringBuilder(serverUrl);
+ if(data.length > 0) {
+ finalUrl.append("?");
+ }
+ for(int i = 0; i < data.length; i++) {
+ if(i == 0) {
+ finalUrl.append("?");
+ } else {
+ finalUrl.append("&");
+ }
+ finalUrl.append(formatter.format(data[i]));
+ }
+
+ GetMethod method = new GetMethod(finalUrl.toString());
+
+ int statusCode = client.executeMethod(method);
+ if (statusCode != HttpStatus.SC_OK) {
+ throw new IOException("Method failed: " + method.getStatusLine());
+ }
+ String responseString = method.getResponseBodyAsString();
+ if(!responseString.startsWith(OK_RESPONSE_PREFIX)) {
+ if(responseString.startsWith(ResourceFileLocationDBServlet.NO_LOCATION_PREFIX)) {
+ return null;
+ }
+ throw new IOException(responseString);
+ }
+ return responseString.substring(OK_RESPONSE_PREFIX.length()+1);
+ }
+
+ private void doPostMethod(final String operation, final String arcName,
+ final String arcUrl)
+ throws IOException {
+ PostMethod method = new PostMethod(serverUrl);
+ NameValuePair[] data = {
+ new NameValuePair(ResourceFileLocationDBServlet.OPERATION_ARGUMENT,
+ operation),
+ new NameValuePair(ResourceFileLocationDBServlet.NAME_ARGUMENT,
+ arcName),
+ new NameValuePair(ResourceFileLocationDBServlet.URL_ARGUMENT,
+ arcUrl)
+ };
+ method.setRequestBody(data);
+ int statusCode = client.executeMethod(method);
+ if (statusCode != HttpStatus.SC_OK) {
+ throw new IOException("Method failed: " + method.getStatusLine());
+ }
+ String responseString = method.getResponseBodyAsString();
+ if(!responseString.startsWith(OK_RESPONSE_PREFIX)) {
+ throw new IOException(responseString);
+ }
+ }
+
+ /* (non-Javadoc)
+ * @see org.archive.wayback.resourcestore.locationdb.ResourceFileLocationDB#shutdown()
+ */
+ public void shutdown() throws IOException {
+ // NO-OP
+ }
+
+ private static void USAGE(String message) {
+ System.err.print("USAGE: " + message + "\n" +
+ "\t[lookup|add|remove|sync] ...\n" +
+ "\n" +
+ "\t lookup LOCATION-DB-URL ARC\n" +
+ "\t\temit all known URLs for arc ARC\n" +
+ "\n" +
+ "\t add LOCATION-DB-URL ARC URL\n" +
+ "\t\tinform locationDB that ARC is located at URL\n" +
+ "\n" +
+ "\t remove LOCATION-DB-URL ARC URL\n" +
+ "\t\tremove reference to ARC at URL in locationDB\n" +
+ "\n" +
+ "\t sync LOCATION-DB-URL DIR DIR-URL\n" +
+ "\t\tscan directory DIR, and submit all ARC files therein\n" +
+ "\t\tto locationDB at url DIR-URL/ARC\n" +
+ "\n" +
+ "\t get-mark LOCATION-DB-URL\n" +
+ "\t\temit an identifier for the current marker in the \n" +
+ "\t\tlocationDB log. These identifiers can be used with the\n" +
+ "\t\tmark-range operation.\n" +
+ "\n" +
+ "\t mark-range LOCATION-DB-URL START END\n" +
+ "\t\temit to STDOUT one line with the name of all ARC files\n" +
+ "\t\tadded to the locationDB between marks START and END\n" +
+ "\n" +
+ "\t add-stream LOCATION-DB-URL\n" +
+ "\t\tread lines from STDIN formatted like:\n" +
+ "\t\t\tNAME<SPACE>URL\n" +
+ "\t\tand for each line, inform locationDB that file NAME is\n" +
+ "\t\tlocated at URL\n"
+ );
+ System.exit(2);
+ }
+
+ /**
+ * @param args
+ */
+ public static void main(String[] args) {
+ if(args.length < 2) {
+ USAGE("");
+ System.exit(1);
+ }
+ String operation = args[0];
+ String dbUrl = args[1];
+ if(!dbUrl.startsWith("http://")) {
+ USAGE("URL argument 1 must begin with http://");
+ }
+
+ RemoteResourceFileLocationDB locationClient =
+ new RemoteResourceFileLocationDB(dbUrl);
+
+ if(operation.equalsIgnoreCase("add-stream")) {
+ BufferedReader r = new BufferedReader(
+ new InputStreamReader(System.in));
+ String line;
+ try {
+ while((line = r.readLine()) != null) {
+ String parts[] = line.split(" ");
+ if(parts.length != 2) {
+ System.err.println("Bad input(" + line + ")");
+ System.exit(2);
+ }
+ locationClient.addNameUrl(parts[0],parts[1]);
+ System.out.println("Added\t" + parts[0] + "\t" + parts[1]);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+
+ } else {
+ if(args.length < 3) {
+ USAGE("");
+ System.exit(1);
+ }
+ String name = args[2];
+ if(operation.equalsIgnoreCase("lookup")) {
+ if(args.length < 3) {
+ USAGE("lookup LOCATION-URL ARC");
+ }
+ try {
+ String[] locations = locationClient.nameToUrls(name);
+ if(locations == null) {
+ System.err.println("No locations for " + name);
+ System.exit(1);
+ }
+ for(int i=0; i <locations.length; i++) {
+ System.out.println(locations[i]);
+ }
+ } catch (IOException e) {
+ System.err.println(e.getMessage());
+ System.exit(1);
+ }
+
+ } else if(operation.equalsIgnoreCase("get-mark")) {
+ if(args.length != 2) {
+ USAGE("get-mark LOCATION-URL");
+ }
+ try {
+ long mark = locationClient.getCurrentMark();
+ System.out.println(mark);
+ } catch (IOException e) {
+ System.err.println(e.getMessage());
+ System.exit(1);
+ }
+
+ } else if(operation.equalsIgnoreCase("mark-range")) {
+ if(args.length != 4) {
+ USAGE("mark-range LOCATION-URL START END");
+ }
+ long start = Long.parseLong(args[3]);
+ long end = Long.parseLong(args[4]);
+ try {
+ Iterator<String> it =
+ locationClient.getNamesBetweenMarks(start,end);
+ while(it.hasNext()) {
+ String next = (String) it.next();
+ System.out.println(next);
+ }
+ } catch (IOException e) {
+ System.err.println(e.getMessage());
+ System.exit(1);
+ }
+
+
+ } else if(operation.equalsIgnoreCase("add")) {
+ if(args.length != 4) {
+ USAGE("add LOCATION-URL ARC ARC-URL");
+ }
+ String url = args[3];
+ if(!url.startsWith("http://")) {
+ USAGE("ARC-URL argument 4 must begin with http://");
+ }
+ try {
+ locationClient.addNameUrl(name,url);
+ System.out.println("OK");
+ } catch (IOException e) {
+ System.err.println(e.getMessage());
+ System.exit(1);
+ }
+
+ } else if(operation.equalsIgnoreCase("remove")) {
+
+ if(args.length != 4) {
+ USAGE("remove LOCATION-URL FILE-NAME FILE-URL");
+ }
+ String url = args[3];
+ if(!url.startsWith("http://")) {
+ USAGE("URL argument 4 must begin with http://");
+ }
+ try {
+ locationClient.removeNameUrl(name,url);
+ System.out.println("OK");
+ } catch (IOException e) {
+ System.err.println(e.getMessage());
+ System.exit(1);
+ }
+
+ } else if(operation.equalsIgnoreCase("sync")) {
+
+ if(args.length != 4) {
+ USAGE("sync LOCATION-URL DIR DIR-URL");
+ }
+ File dir = new File(name);
+ String dirUrl = args[3];
+ if(!dirUrl.startsWith("http://")) {
+ USAGE("DIR-URL argument 4 must begin with http://");
+ }
+ try {
+ if(!dir.isDirectory()) {
+ USAGE("DIR " + name + " is not a directory");
+ }
+
+ FileFilter filter = new FileFilter() {
+ public boolean accept(File daFile) {
+ return daFile.isFile() &&
+ (daFile.getName().endsWith(ARC_SUFFIX) ||
+ daFile.getName().endsWith(ARC_GZ_SUFFIX) ||
+ daFile.getName().endsWith(WARC_SUFFIX) ||
+ daFile.getName().endsWith(WARC_GZ_SUFFIX));
+ }
+ };
+
+ File[] files = dir.listFiles(filter);
+ if(files == null) {
+ throw new IOException("Directory " + dir.getAbsolutePath() +
+ " is not a directory or had an IO error");
+ }
+ for(int i = 0; i < files.length; i++) {
+ File file = files[i];
+ String fileName = file.getName();
+ String fileUrl = dirUrl + fileName;
+ LOGGER.info("Adding location " + fileUrl +
+ " for file " + fileName);
+ locationClient.addNameUrl(fileName,fileUrl);
+ }
+ } catch (IOException e) {
+ System.err.println(e.getMessage());
+ System.exit(1);
+ }
+
+ } else {
+ USAGE(" unknown operation " + operation);
+ }
+ }
+ }
+}
Deleted: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/ResourceFileLocationDB.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/ResourceFileLocationDB.java 2008-06-05 21:37:36 UTC (rev 2285)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/ResourceFileLocationDB.java 2008-06-05 21:52:54 UTC (rev 2286)
@@ -1,305 +0,0 @@
-/* FileLocationDB
- *
- * $Id$
- *
- * Created on 3:08:59 PM Aug 18, 2006.
- *
- * Copyright (C) 2006 Internet Archive.
- *
- * This file is part of Wayback.
- *
- * Wayback is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Lesser Public License as published by
- * the Free Software Foundation; either version 2.1 of the License, or
- * any later version.
- *
- * Wayback is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Lesser Public License for more details.
- *
- * You should have received a copy of the GNU Lesser Public License
- * along with Wayback; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
- */
-package org.archive.wayback.resourcestore.locationdb;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-
-import org.archive.wayback.bdb.BDBRecordSet;
-import org.archive.wayback.exception.ConfigurationException;
-import org.archive.wayback.util.CloseableIterator;
-
-import com.sleepycat.je.DatabaseException;
-
-/**
- *
- *
- * @author brad
- * @version $Date$, $Revision$
- */
-public class ResourceFileLocationDB extends BDBRecordSet {
-
- /**
- * String id for implementation class of FileLocationDBs.
- */
- public static final String FILE_LOCATION_DB_CLASS = "filelocationdb";
-
- protected static final String ARC_DB_PATH = "filelocationdb.path";
-
- protected static final String ARC_DB_NAME = "filelocationdb.name";
-
- protected static final String ARC_DB_LOG = "filelocationdb.logpath";
-
- private final static String urlDelimiter = " ";
-
- private final static String urlDelimiterRE = " ";
-
- private ResourceFileLocationDBLog log;
- private String logPath = null;
- private String bdbPath = null;
- private String bdbName = null;
-
- /**
- * Constructor
- */
- public ResourceFileLocationDB() {
- super();
- }
-
- /**
- * @throws DatabaseException
- * @throws ConfigurationException
- */
- public void init() throws DatabaseException, ConfigurationException {
- if(logPath == null) {
- throw new ConfigurationException("No logPath");
- }
- log = new ResourceFileLocationDBLog(logPath);
- initializeDB(bdbPath,bdbName);
- }
-
- /**
- * return an array of String URLs for all known locations of the ARC file
- * in the DB.
- * @param arcName
- * @return String[] of URLs to arcName
- * @throws DatabaseException
- */
- public String[] arcToUrls(final String arcName) throws DatabaseException {
-
- String[] arcUrls = null;
- String valueString = get(arcName);
- if(valueString != null && valueString.length() > 0) {
- arcUrls = valueString.split(urlDelimiterRE);
- }
- return arcUrls;
- }
-
- /**
- * add an Url location for an arcName, unless it already exists
- * @param arcName
- * @param arcUrl
- * @throws DatabaseException
- * @throws IOException
- */
- public void addArcUrl(final String arcName, final String arcUrl) throws DatabaseException, IOException {
-
- // need to first see if there is already an entry for this arcName.
- // if not, add arcUrl as the value.
- // if so, check the current arcUrl locations for arcName
- // if arcUrl exists, do nothing
- // if arcUrl does not exist, add, and set that as the value.
-
- String newValue = null;
- String oldValue = get(arcName);
- if(oldValue != null && oldValue.length() > 0) {
- String curUrls[] = oldValue.split(urlDelimiterRE);
- boolean found = false;
- for(int i=0; i < curUrls.length; i++) {
- if(arcUrl.equals(curUrls[i])) {
- found = true;
- break;
- }
- }
- if(found == false) {
- newValue = oldValue + " " + arcUrl;
- }
- } else {
- // null or empty value
- newValue = arcUrl;
- if(oldValue == null) log.addArc(arcName);
- }
-
- // did we find a value?
- if(newValue != null) {
- put(arcName,newValue);
- }
- }
-
- /**
- * remove a single Url location for an arcName, if it exists
- * @param arcName
- * @param arcUrl
- * @throws DatabaseException
- */
- public void removeArcUrl(final String arcName, final String arcUrl) throws DatabaseException {
- // need to first see if there is already an entry for this arcName.
- // if not, do nothing
- // if so, loop thru all current arcUrl locations for arcName
- // keep any that are not arcUrl
- // if any locations are left, update to the new value, sans arcUrl
- // if none are left, remove the entry from the db
-
- StringBuilder newValue = new StringBuilder();
- String oldValue = get(arcName);
- if(oldValue != null && oldValue.length() > 0) {
- String curUrls[] = oldValue.split(urlDelimiterRE);
-
- for(int i=0; i < curUrls.length; i++) {
- if(!arcUrl.equals(curUrls[i])) {
- if(newValue.length() > 0) {
- newValue.append(urlDelimiter);
- }
- newValue.append(curUrls[i]);
- }
- }
-
- if(newValue.length() > 0) {
-
- // update
- put(arcName, newValue.toString());
-
- } else {
-
- // remove the entry:
- delete(arcName);
- }
- }
- }
-
- /**
- * @param start
- * @param end
- * @return Iterator for traversing arcs between start and end.
- * @throws IOException
- */
- public CloseableIterator<String> getArcsBetweenMarks(long start, long end)
- throws IOException {
- return log.getArcsBetweenMarks(start, end);
- }
-
- /**
- * @return current "Mark" for the log. Currently, it's just the length of
- * the log file.
- */
- public long getCurrentMark() {
- return log.getCurrentMark();
- }
-
- /**
- * @return the logPath
- */
- public String getLogPath() {
- return logPath;
- }
-
- /**
- * @param logPath the logPath to set
- */
- public void setLogPath(String logPath) {
- this.logPath = logPath;
- }
-
- /**
- * @return the bdbPath
- */
- public String getBdbPath() {
- return bdbPath;
- }
-
- /**
- * @param bdbPath the bdbPath to set
- */
- public void setBdbPath(String bdbPath) {
- this.bdbPath = bdbPath;
- }
-
- /**
- * @return the bdbName
- */
- public String getBdbName() {
- return bdbName;
- }
-
- /**
- * @param bdbName the bdbName to set
- */
- public void setBdbName(String bdbName) {
- this.bdbName = bdbName;
- }
- private static void USAGE(String message) {
- System.err.print("USAGE: " + message + "\n" +
- "\tDBDIR DBNAME LOGPATH\n" +
- "\n" +
- "\t\tread lines from STDIN formatted like:\n" +
- "\t\t\tNAME<SPACE>URL\n" +
- "\t\tand for each line, add to locationDB that file NAME is\n" +
- "\t\tlocated at URL. Use locationDB in DBDIR at DBNAME, \n" +
- "\t\tcreating if it does not exist.\n"
- );
- System.exit(2);
- }
-
- /**
- * @param args
- */
- public static void main(String[] args) {
- if(args.length != 3) {
- USAGE("");
- System.exit(1);
- }
- String bdbPath = args[0];
- String bdbName = args[1];
- String logPath = args[2];
- ResourceFileLocationDB db = new ResourceFileLocationDB();
- db.setBdbPath(bdbPath);
- db.setBdbName(bdbName);
- db.setLogPath(logPath);
- BufferedReader r = new BufferedReader(
- new InputStreamReader(System.in));
- String line;
- int exitCode = 0;
- try {
- db.init();
- while((line = r.readLine()) != null) {
- String parts[] = line.split(" ");
- if(parts.length != 2) {
- System.err.println("Bad input(" + line + ")");
- System.exit(2);
- }
- db.addArcUrl(parts[0],parts[1]);
- System.out.println("Added\t" + parts[0] + "\t" + parts[1]);
- }
- } catch (IOException e) {
- e.printStackTrace();
- exitCode = 1;
- } catch (DatabaseException e) {
- e.printStackTrace();
- exitCode = 1;
- } catch (ConfigurationException e) {
- e.printStackTrace();
- exitCode = 1;
- } finally {
- try {
- db.shutdownDB();
- } catch (DatabaseException e) {
- e.printStackTrace();
- exitCode = 1;
- }
- }
- System.exit(exitCode);
- }
-}
Deleted: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/ResourceFileLocationDBClient.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/ResourceFileLocationDBClient.java 2008-06-05 21:37:36 UTC (rev 2285)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/ResourceFileLocationDBClient.java 2008-06-05 21:52:54 UTC (rev 2286)
@@ -1,416 +0,0 @@
-/* FileLocationDBClient
- *
- * $Id$
- *
- * Created on 5:59:49 PM Aug 21, 2006.
- *
- * Copyright (C) 2006 Internet Archive.
- *
- * This file is part of Wayback.
- *
- * Wayback is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Lesser Public License as published by
- * the Free Software Foundation; either version 2.1 of the License, or
- * any later version.
- *
- * Wayback is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Lesser Public License for more details.
- *
- * You should have received a copy of the GNU Lesser Public License
- * along with Wayback; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
- */
-package org.archive.wayback.resourcestore.locationdb;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileFilter;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.logging.Logger;
-
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpStatus;
-import org.apache.commons.httpclient.NameValuePair;
-import org.apache.commons.httpclient.methods.GetMethod;
-import org.apache.commons.httpclient.methods.PostMethod;
-import org.apache.commons.httpclient.util.ParameterFormatter;
-import org.archive.wayback.resourcestore.locationdb.ResourceFileLocationDBServlet;
-
-/**
- *
- *
- * @author brad
- * @version $Date$, $Revision$
- */
-public class ResourceFileLocationDBClient {
- private static final Logger LOGGER = Logger.getLogger(ResourceFileLocationDBClient
- .class.getName());
-
- private final static String ARC_SUFFIX = ".arc";
- private final static String ARC_GZ_SUFFIX = ".arc.gz";
- private final static String WARC_SUFFIX = ".warc";
- private final static String WARC_GZ_SUFFIX = ".warc.gz";
- private final static String OK_RESPONSE_PREFIX = "OK ";
- private HttpClient client = null;
-
- private String serverUrl = null;
-
- /**
- * @param serverUrl
- */
- public ResourceFileLocationDBClient(final String serverUrl) {
- super();
- this.serverUrl = serverUrl;
- this.client = new HttpClient();
- }
-
- /**
- * @return long value representing the current end "mark" of the db log
- * @throws NumberFormatException
- * @throws IOException
- */
- public long getCurrentMark() throws NumberFormatException, IOException {
- NameValuePair[] args = {
- new NameValuePair(
- ResourceFileLocationDBServlet.OPERATION_ARGUMENT,
- ResourceFileLocationDBServlet.GETMARK_OPERATION),
- };
- return Long.parseLong(doGetMethod(args));
- }
-
- /**
- * @param start
- * @param end
- * @return Iterator of arc file names between marks start and end
- * @throws IOException
- */
- public Iterator<String> getArcsBetweenMarks(long start, long end)
- throws IOException {
- NameValuePair[] args = {
- new NameValuePair(
- ResourceFileLocationDBServlet.OPERATION_ARGUMENT,
- ResourceFileLocationDBServlet.GETRANGE_OPERATION),
- new NameValuePair(
- ResourceFileLocationDBServlet.START_ARGUMENT,
- String.valueOf(start)),
- new NameValuePair(
- ResourceFileLocationDBServlet.END_ARGUMENT,
- String.valueOf(end))
- };
- return Arrays.asList(doGetMethod(args).split("\n")).iterator();
- }
-
- /**
- * return an array of String URLs for all known locations of the ARC file
- * in the DB.
- * @param arcName
- * @return String[] of URLs to arcName
- * @throws IOException
- */
- public String[] arcToUrls(final String arcName) throws IOException {
-
- NameValuePair[] args = {
- new NameValuePair(
- ResourceFileLocationDBServlet.OPERATION_ARGUMENT,
- ResourceFileLocationDBServlet.LOOKUP_OPERATION),
-
- new NameValuePair(
- ResourceFileLocationDBServlet.NAME_ARGUMENT,
- arcName)
- };
- String locations = doGetMethod(args);
- if(locations != null) {
- return locations.split("\n");
- }
- return null;
- }
-
-
- /**
- * add an Url location for an arcName, unless it already exists
- * @param arcName
- * @param arcUrl
- * @throws IOException
- */
- public void addArcUrl(final String arcName, final String arcUrl)
- throws IOException {
- doPostMethod(ResourceFileLocationDBServlet.ADD_OPERATION, arcName, arcUrl);
- }
-
- /**
- * remove a single Url location for an arcName, if it exists
- * @param arcName
- * @param arcUrl
- * @throws IOException
- */
- public void removeArcUrl(final String arcName, final String arcUrl)
- throws IOException {
- doPostMethod(ResourceFileLocationDBServlet.REMOVE_OPERATION, arcName, arcUrl);
- }
-
- private String doGetMethod(NameValuePair[] data) throws IOException {
- ParameterFormatter formatter = new ParameterFormatter();
- formatter.setAlwaysUseQuotes(false);
- StringBuilder finalUrl = new StringBuilder(serverUrl);
- if(data.length > 0) {
- finalUrl.append("?");
- }
- for(int i = 0; i < data.length; i++) {
- if(i == 0) {
- finalUrl.append("?");
- } else {
- finalUrl.append("&");
- }
- finalUrl.append(formatter.format(data[i]));
- }
-
- GetMethod method = new GetMethod(finalUrl.toString());
-
- int statusCode = client.executeMethod(method);
- if (statusCode != HttpStatus.SC_OK) {
- throw new IOException("Method failed: " + method.getStatusLine());
- }
- String responseString = method.getResponseBodyAsString();
- if(!responseString.startsWith(OK_RESPONSE_PREFIX)) {
- if(responseString.startsWith(ResourceFileLocationDBServlet.NO_LOCATION_PREFIX)) {
- return null;
- }
- throw new IOException(responseString);
- }
- return responseString.substring(OK_RESPONSE_PREFIX.length()+1);
- }
-
- private void doPostMethod(final String operation, final String arcName,
- final String arcUrl)
- throws IOException {
- PostMethod method = new PostMethod(serverUrl);
- NameValuePair[] data = {
- new NameValuePair(ResourceFileLocationDBServlet.OPERATION_ARGUMENT,
- operation),
- new NameValuePair(ResourceFileLocationDBServlet.NAME_ARGUMENT,
- arcName),
- new NameValuePair(ResourceFileLocationDBServlet.URL_ARGUMENT,
- arcUrl)
- };
- method.setRequestBody(data);
- int statusCode = client.executeMethod(method);
- if (statusCode != HttpStatus.SC_OK) {
- throw new IOException("Method failed: " + method.getStatusLine());
- }
- String responseString = method.getResponseBodyAsString();
- if(!responseString.startsWith(OK_RESPONSE_PREFIX)) {
- throw new IOException(responseString);
- }
- }
-
- private static void USAGE(String message) {
- System.err.print("USAGE: " + message + "\n" +
- "\t[lookup|add|remove|sync] ...\n" +
- "\n" +
- "\t lookup LOCATION-DB-URL ARC\n" +
- "\t\temit all known URLs for arc ARC\n" +
- "\n" +
- "\t add LOCATION-DB-URL ARC URL\n" +
- "\t\tinform locationDB that ARC is located at URL\n" +
- "\n" +
- "\t remove LOCATION-DB-URL ARC URL\n" +
- "\t\tremove reference to ARC at URL in locationDB\n" +
- "\n" +
- "\t sync LOCATION-DB-URL DIR DIR-URL\n" +
- "\t\tscan directory DIR, and submit all ARC files therein\n" +
- "\t\tto locationDB at url DIR-URL/ARC\n" +
- "\n" +
- "\t get-mark LOCATION-DB-URL\n" +
- "\t\temit an identifier for the current marker in the \n" +
- "\t\tlocationDB log. These identifiers can be used with the\n" +
- "\t\tmark-range operation.\n" +
- "\n" +
- "\t mark-range LOCATION-DB-URL START END\n" +
- "\t\temit to STDOUT one line with the name of all ARC files\n" +
- "\t\tadded to the locationDB between marks START and END\n" +
- "\n" +
- "\t add-stream LOCATION-DB-URL\n" +
- "\t\tread lines from STDIN formatted like:\n" +
- "\t\t\tNAME<SPACE>URL\n" +
- "\t\tand for each line, inform locationDB that file NAME is\n" +
- "\t\tlocated at URL\n"
- );
- System.exit(2);
- }
-
- /**
- * @param args
- */
- public static void main(String[] args) {
- if(args.length < 2) {
- USAGE("");
- System.exit(1);
- }
- String operation = args[0];
- String url = args[1];
- if(!url.startsWith("http://")) {
- USAGE("URL argument 1 must begin with http://");
- }
-
- ResourceFileLocationDBClient locationClient = new ResourceFileLocationDBClient(url);
-
- if(operation.equalsIgnoreCase("add-stream")) {
- BufferedReader r = new BufferedReader(
- new InputStreamReader(System.in));
- String line;
- try {
- while((line = r.readLine()) != null) {
- String parts[] = line.split(" ");
- if(parts.length != 2) {
- System.err.println("Bad input(" + line + ")");
- System.exit(2);
- }
- locationClient.addArcUrl(parts[0],parts[1]);
- System.out.println("Added\t" + parts[0] + "\t" + parts[1]);
- }
- } catch (IOException e) {
- e.printStackTrace();
- System.exit(1);
- }
-
- } else {
- if(args.length < 3) {
- USAGE("");
- System.exit(1);
- }
- String arc = args[2];
- if(operation.equalsIgnoreCase("lookup")) {
- if(args.length < 3) {
- USAGE("lookup LOCATION-URL ARC");
- }
- try {
- String[] locations = locationClient.arcToUrls(arc);
- if(locations == null) {
- System.err.println("No locations for " + arc);
- System.exit(1);
- }
- for(int i=0; i <locations.length; i++) {
- System.out.println(locations[i]);
- }
- } catch (IOException e) {
- System.err.println(e.getMessage());
- System.exit(1);
- }
-
- } else if(operation.equalsIgnoreCase("get-mark")) {
- if(args.length != 2) {
- USAGE("get-mark LOCATION-URL");
- }
- try {
- long mark = locationClient.getCurrentMark();
- System.out.println(mark);
- } catch (IOException e) {
- System.err.println(e.getMessage());
- System.exit(1);
- }
-
- } else if(operation.equalsIgnoreCase("mark-range")) {
- if(args.length != 4) {
- USAGE("mark-range LOCATION-URL START END");
- }
- long start = Long.parseLong(args[3]);
- long end = Long.parseLong(args[4]);
- try {
- Iterator<String> it =
- locationClient.getArcsBetweenMarks(start,end);
- while(it.hasNext()) {
- String next = (String) it.next();
- System.out.println(next);
- }
- } catch (IOException e) {
- System.err.println(e.getMessage());
- System.exit(1);
- }
-
-
- } else if(operation.equalsIgnoreCase("add")) {
- if(args.length != 4) {
- USAGE("add LOCATION-URL ARC ARC-URL");
- }
- String arcUrl = args[3];
- if(!arcUrl.startsWith("http://")) {
- USAGE("ARC-URL argument 4 must begin with http://");
- }
- try {
- locationClient.addArcUrl(arc,arcUrl);
- System.out.println("OK");
- } catch (IOException e) {
- System.err.println(e.getMessage());
- System.exit(1);
- }
-
- } else if(operation.equalsIgnoreCase("remove")) {
-
- if(args.length != 4) {
- USAGE("remove LOCATION-URL ARC ARC-URL");
- }
- String arcUrl = args[3];
- if(!arcUrl.startsWith("http://")) {
- USAGE("ARC-URL argument 4 must begin with http://");
- }
- try {
- locationClient.removeArcUrl(arc,arcUrl);
- System.out.println("OK");
- } catch (IOException e) {
- System.err.println(e.getMessage());
- System.exit(1);
- }
-
- } else if(operation.equalsIgnoreCase("sync")) {
-
- if(args.length != 4) {
- USAGE("sync LOCATION-URL DIR DIR-URL");
- }
- File dir = new File(arc);
- String dirUrl = args[3];
- if(!dirUrl.startsWith("http://")) {
- USAGE("DIR-URL argument 4 must begin with http://");
- }
- try {
- if(!dir.isDirectory()) {
- USAGE("DIR " + arc + " is not a directory");
- }
-
- FileFilter filter = new FileFilter() {
- public boolean accept(File daFile) {
- return daFile.isFile() &&
- (daFile.getName().endsWith(ARC_SUFFIX) ||
- daFile.getName().endsWith(ARC_GZ_SUFFIX) ||
- daFile.getName().endsWith(WARC_SUFFIX) ||
- daFile.getName().endsWith(WARC_GZ_SUFFIX));
- }
- };
-
- File[] files = dir.listFiles(filter);
- if(files == null) {
- throw new IOException("Directory " + dir.getAbsolutePath() +
- " is not a directory or had an IO error");
- }
- for(int i = 0; i < files.length; i++) {
- File file = files[i];
- String name = file.getName();
- String fileUrl = dirUrl + name;
- LOGGER.info("Adding location " + fileUrl + " for file " + name);
- locationClient.addArcUrl(name,fileUrl);
- }
- } catch (IOException e) {
- System.err.println(e.getMessage());
- System.exit(1);
- }
-
- } else {
- USAGE(" unknown operation " + operation);
- }
- }
- }
-}
Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/ResourceFileLocationDBLog.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/ResourceFileLocationDBLog.java 2008-06-05 21:37:36 UTC (rev 2285)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/ResourceFileLocationDBLog.java 2008-06-05 21:52:54 UTC (rev 2286)
@@ -38,7 +38,10 @@
import sun.reflect.generics.reflectiveObjects.NotImplementedException;
/**
+ * Simple log file tracking new names being added to a ResourceFileLocationDB.
*
+ * Also supports returning an iterator of Strings to a byte range of the log, to
+ * simplify tracking deltas to the DB.
*
* @author brad
* @version $Date$, $Revision$
Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/ResourceFileLocationDBServlet.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/ResourceFileLocationDBServlet.java 2008-06-05 21:37:36 UTC (rev 2285)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/ResourceFileLocationDBServlet.java 2008-06-05 21:52:54 UTC (rev 2286)
@@ -34,14 +34,13 @@
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import org.archive.wayback.resourcestore.locationdb.ResourceFileLocationDB;
import org.archive.wayback.webapp.ServletRequestContext;
-import com.sleepycat.je.DatabaseException;
-
/**
+ * ServletRequestContext enabling remote HTTP GET/POST access to a local
+ * ResourceFileLocationDB. See RemoveResourceFileLocationDB for the client
+ * class implemented against this.
*
- *
* @author brad
* @version $Date$, $Revision$
*/
@@ -68,9 +67,8 @@
@SuppressWarnings("unchecked")
Map<String,String[]> queryMap = httpRequest.getParameterMap();
String message;
- ResourceFileLocationDB locationDB = getLocationDB();
try {
- message = handleOperation(locationDB,queryMap);
+ message = handleOperation(queryMap);
httpResponse.setStatus(HttpServletResponse.SC_OK);
httpResponse.setContentType("text/plain");
OutputStream os = httpResponse.getOutputStream();
@@ -83,18 +81,17 @@
return true;
}
- private String handleOperation(ResourceFileLocationDB locationDB,
- Map<String,String[]> queryMap)
+ private String handleOperation(Map<String,String[]> queryMap)
throws ParseException {
String operation = getRequiredMapParam(queryMap, OPERATION_ARGUMENT);
String message;
try {
if (operation.equals(LOOKUP_OPERATION)) {
- String arcName = getRequiredMapParam(queryMap, NAME_ARGUMENT);
+ String name = getRequiredMapParam(queryMap, NAME_ARGUMENT);
- message = NO_LOCATION_PREFIX + " " + arcName;
- String arcUrls[] = locationDB.arcToUrls(arcName);
+ message = NO_LOCATION_PREFIX + " " + name;
+ String arcUrls[] = locationDB.nameToUrls(name);
if (arcUrls != null && arcUrls.length > 0) {
StringBuffer buf = new StringBuffer("OK ");
for (int i = 0; i < arcUrls.length; i++) {
@@ -112,7 +109,7 @@
long start = Long.parseLong(getRequiredMapParam(queryMap, START_ARGUMENT));
long end = Long.parseLong(getRequiredMapParam(queryMap, END_ARGUMENT));
- Iterator<String> itr = locationDB.getArcsBetweenMarks(start,end);
+ Iterator<String> itr = locationDB.getNamesBetweenMarks(start,end);
StringBuilder str = new StringBuilder();
str.append("OK ");
while(itr.hasNext()) {
@@ -123,17 +120,17 @@
} else {
- String arcName = getRequiredMapParam(queryMap, NAME_ARGUMENT);
- String arcUrl = getRequiredMapParam(queryMap, URL_ARGUMENT);
+ String name = getRequiredMapParam(queryMap, NAME_ARGUMENT);
+ String url = getRequiredMapParam(queryMap, URL_ARGUMENT);
if (operation.equals(ADD_OPERATION)) {
- locationDB.addArcUrl(arcName, arcUrl);
- message = "OK added url " + arcUrl + " for " + arcName;
+ locationDB.addNameUrl(name, url);
+ message = "OK added url " + url + " for " + name;
} else if (operation.equals(REMOVE_OPERATION)) {
- getLocationDB().removeArcUrl(arcName, arcUrl);
- message = "OK removed url " + arcUrl + " for " + arcName;
+ locationDB.removeNameUrl(name, url);
+ message = "OK removed url " + url + " for " + name;
} else {
@@ -143,11 +140,7 @@
}
}
- } catch (DatabaseException e) {
- e.printStackTrace();
- message = e.getMessage();
} catch (IOException e) {
- // TODO Auto-generated catch block
e.printStackTrace();
message = e.getMessage();
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
