Menu
▾
▴
[Archive-access-cvs] SF.net SVN: archive-access: [2282] trunk/archive-access/projects/wayback/ wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb
|
From: <bra...@us...> - 2008-06-05 20:38:48
|
Revision: 2282
http://archive-access.svn.sourceforge.net/archive-access/?rev=2282&view=rev
Author: bradtofel
Date: 2008-06-05 13:38:55 -0700 (Thu, 05 Jun 2008)
Log Message:
-----------
MOVED: change package name
Added Paths:
-----------
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/ArcProxyServlet.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileLocationDB.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileLocationDBClient.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileLocationDBLog.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileLocationDBServlet.java
Copied: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/ArcProxyServlet.java (from rev 2055, trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/ArcProxyServlet.java)
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/ArcProxyServlet.java (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/ArcProxyServlet.java 2008-06-05 20:38:55 UTC (rev 2282)
@@ -0,0 +1,124 @@
+/* ArcProxyServlet
+ *
+ * $Id$
+ *
+ * Created on 6:19:54 PM Aug 10, 2006.
+ *
+ * Copyright (C) 2006 Internet Archive.
+ *
+ * This file is part of Wayback.
+ *
+ * Wayback is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * any later version.
+ *
+ * Wayback is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser Public License
+ * along with Wayback; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+package org.archive.wayback.resourcestore.locationdb;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URL;
+import java.net.URLConnection;
+import java.text.ParseException;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.archive.wayback.resourcestore.locationdb.FileLocationDB;
+import org.archive.wayback.webapp.ServletRequestContext;
+
+import com.sleepycat.je.DatabaseException;
+
+/**
+ *
+ *
+ * @author brad
+ * @version $Date$, $Revision$
+ */
+public class ArcProxyServlet extends ServletRequestContext {
+
+ private static final String RANGE_HTTP_HEADER = "Range";
+ private static final String CONTENT_TYPE_HEADER = "Content-Type";
+ private static final String CONTENT_TYPE = "application/x-gzip";
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+ private FileLocationDB locationDB = null;
+
+ public boolean handleRequest(HttpServletRequest httpRequest,
+ HttpServletResponse httpResponse) throws IOException,
+ ServletException {
+
+ try {
+ String arc = httpRequest.getRequestURI();
+ arc = arc.substring(arc.lastIndexOf('/')+1);
+ if(arc.length() == 0) {
+ throw new ParseException("no/invalid arc",0);
+ }
+ String urls[] = locationDB.arcToUrls(arc);
+ if(urls == null || urls.length == 0) {
+ throw new DatabaseException("Unable to locate("+arc+")");
+ }
+ String urlString = urls[0];
+ String rangeHeader = httpRequest.getHeader(RANGE_HTTP_HEADER);
+ URL url = new URL(urlString);
+ URLConnection conn = url.openConnection();
+ if(rangeHeader != null) {
+ conn.addRequestProperty(RANGE_HTTP_HEADER,rangeHeader);
+ }
+ InputStream is = conn.getInputStream();
+ httpResponse.setStatus(HttpServletResponse.SC_OK);
+ String typeHeader = conn.getHeaderField(CONTENT_TYPE_HEADER);
+ if(typeHeader == null) {
+ typeHeader = CONTENT_TYPE;
+ }
+ httpResponse.setContentType(typeHeader);
+ OutputStream os = httpResponse.getOutputStream();
+ int BUF_SIZE = 4096;
+ byte[] buffer = new byte[BUF_SIZE];
+ try {
+ for (int r = -1; (r = is.read(buffer, 0, BUF_SIZE)) != -1;) {
+ os.write(buffer, 0, r);
+ }
+ } finally {
+ is.close();
+ }
+ } catch (ParseException e) {
+ e.printStackTrace();
+ httpResponse.sendError(HttpServletResponse.SC_BAD_REQUEST,
+ e.getMessage());
+ } catch (DatabaseException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ httpResponse.sendError(HttpServletResponse.SC_NOT_FOUND,
+ e.getMessage());
+ }
+ return true;
+ }
+
+ /**
+ * @return the locationDB
+ */
+ public FileLocationDB getLocationDB() {
+ return locationDB;
+ }
+
+ /**
+ * @param locationDB the locationDB to set
+ */
+ public void setLocationDB(FileLocationDB locationDB) {
+ this.locationDB = locationDB;
+ }
+}
Copied: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileLocationDB.java (from rev 2261, trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/FileLocationDB.java)
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileLocationDB.java (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileLocationDB.java 2008-06-05 20:38:55 UTC (rev 2282)
@@ -0,0 +1,305 @@
+/* FileLocationDB
+ *
+ * $Id$
+ *
+ * Created on 3:08:59 PM Aug 18, 2006.
+ *
+ * Copyright (C) 2006 Internet Archive.
+ *
+ * This file is part of Wayback.
+ *
+ * Wayback is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * any later version.
+ *
+ * Wayback is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser Public License
+ * along with Wayback; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+package org.archive.wayback.resourcestore.locationdb;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+import org.archive.wayback.bdb.BDBRecordSet;
+import org.archive.wayback.exception.ConfigurationException;
+import org.archive.wayback.util.CloseableIterator;
+
+import com.sleepycat.je.DatabaseException;
+
+/**
+ *
+ *
+ * @author brad
+ * @version $Date$, $Revision$
+ */
+public class FileLocationDB extends BDBRecordSet {
+
+ /**
+ * String id for implementation class of FileLocationDBs.
+ */
+ public static final String FILE_LOCATION_DB_CLASS = "filelocationdb";
+
+ protected static final String ARC_DB_PATH = "filelocationdb.path";
+
+ protected static final String ARC_DB_NAME = "filelocationdb.name";
+
+ protected static final String ARC_DB_LOG = "filelocationdb.logpath";
+
+ private final static String urlDelimiter = " ";
+
+ private final static String urlDelimiterRE = " ";
+
+ private FileLocationDBLog log;
+ private String logPath = null;
+ private String bdbPath = null;
+ private String bdbName = null;
+
+ /**
+ * Constructor
+ */
+ public FileLocationDB() {
+ super();
+ }
+
+ /**
+ * @throws DatabaseException
+ * @throws ConfigurationException
+ */
+ public void init() throws DatabaseException, ConfigurationException {
+ if(logPath == null) {
+ throw new ConfigurationException("No logPath");
+ }
+ log = new FileLocationDBLog(logPath);
+ initializeDB(bdbPath,bdbName);
+ }
+
+ /**
+ * return an array of String URLs for all known locations of the ARC file
+ * in the DB.
+ * @param arcName
+ * @return String[] of URLs to arcName
+ * @throws DatabaseException
+ */
+ public String[] arcToUrls(final String arcName) throws DatabaseException {
+
+ String[] arcUrls = null;
+ String valueString = get(arcName);
+ if(valueString != null && valueString.length() > 0) {
+ arcUrls = valueString.split(urlDelimiterRE);
+ }
+ return arcUrls;
+ }
+
+ /**
+ * add an Url location for an arcName, unless it already exists
+ * @param arcName
+ * @param arcUrl
+ * @throws DatabaseException
+ * @throws IOException
+ */
+ public void addArcUrl(final String arcName, final String arcUrl) throws DatabaseException, IOException {
+
+ // need to first see if there is already an entry for this arcName.
+ // if not, add arcUrl as the value.
+ // if so, check the current arcUrl locations for arcName
+ // if arcUrl exists, do nothing
+ // if arcUrl does not exist, add, and set that as the value.
+
+ String newValue = null;
+ String oldValue = get(arcName);
+ if(oldValue != null && oldValue.length() > 0) {
+ String curUrls[] = oldValue.split(urlDelimiterRE);
+ boolean found = false;
+ for(int i=0; i < curUrls.length; i++) {
+ if(arcUrl.equals(curUrls[i])) {
+ found = true;
+ break;
+ }
+ }
+ if(found == false) {
+ newValue = oldValue + " " + arcUrl;
+ }
+ } else {
+ // null or empty value
+ newValue = arcUrl;
+ if(oldValue == null) log.addArc(arcName);
+ }
+
+ // did we find a value?
+ if(newValue != null) {
+ put(arcName,newValue);
+ }
+ }
+
+ /**
+ * remove a single Url location for an arcName, if it exists
+ * @param arcName
+ * @param arcUrl
+ * @throws DatabaseException
+ */
+ public void removeArcUrl(final String arcName, final String arcUrl) throws DatabaseException {
+ // need to first see if there is already an entry for this arcName.
+ // if not, do nothing
+ // if so, loop thru all current arcUrl locations for arcName
+ // keep any that are not arcUrl
+ // if any locations are left, update to the new value, sans arcUrl
+ // if none are left, remove the entry from the db
+
+ StringBuilder newValue = new StringBuilder();
+ String oldValue = get(arcName);
+ if(oldValue != null && oldValue.length() > 0) {
+ String curUrls[] = oldValue.split(urlDelimiterRE);
+
+ for(int i=0; i < curUrls.length; i++) {
+ if(!arcUrl.equals(curUrls[i])) {
+ if(newValue.length() > 0) {
+ newValue.append(urlDelimiter);
+ }
+ newValue.append(curUrls[i]);
+ }
+ }
+
+ if(newValue.length() > 0) {
+
+ // update
+ put(arcName, newValue.toString());
+
+ } else {
+
+ // remove the entry:
+ delete(arcName);
+ }
+ }
+ }
+
+ /**
+ * @param start
+ * @param end
+ * @return Iterator for traversing arcs between start and end.
+ * @throws IOException
+ */
+ public CloseableIterator<String> getArcsBetweenMarks(long start, long end)
+ throws IOException {
+ return log.getArcsBetweenMarks(start, end);
+ }
+
+ /**
+ * @return current "Mark" for the log. Currently, it's just the length of
+ * the log file.
+ */
+ public long getCurrentMark() {
+ return log.getCurrentMark();
+ }
+
+ /**
+ * @return the logPath
+ */
+ public String getLogPath() {
+ return logPath;
+ }
+
+ /**
+ * @param logPath the logPath to set
+ */
+ public void setLogPath(String logPath) {
+ this.logPath = logPath;
+ }
+
+ /**
+ * @return the bdbPath
+ */
+ public String getBdbPath() {
+ return bdbPath;
+ }
+
+ /**
+ * @param bdbPath the bdbPath to set
+ */
+ public void setBdbPath(String bdbPath) {
+ this.bdbPath = bdbPath;
+ }
+
+ /**
+ * @return the bdbName
+ */
+ public String getBdbName() {
+ return bdbName;
+ }
+
+ /**
+ * @param bdbName the bdbName to set
+ */
+ public void setBdbName(String bdbName) {
+ this.bdbName = bdbName;
+ }
+ private static void USAGE(String message) {
+ System.err.print("USAGE: " + message + "\n" +
+ "\tDBDIR DBNAME LOGPATH\n" +
+ "\n" +
+ "\t\tread lines from STDIN formatted like:\n" +
+ "\t\t\tNAME<SPACE>URL\n" +
+ "\t\tand for each line, add to locationDB that file NAME is\n" +
+ "\t\tlocated at URL. Use locationDB in DBDIR at DBNAME, \n" +
+ "\t\tcreating if it does not exist.\n"
+ );
+ System.exit(2);
+ }
+
+ /**
+ * @param args
+ */
+ public static void main(String[] args) {
+ if(args.length != 3) {
+ USAGE("");
+ System.exit(1);
+ }
+ String bdbPath = args[0];
+ String bdbName = args[1];
+ String logPath = args[2];
+ FileLocationDB db = new FileLocationDB();
+ db.setBdbPath(bdbPath);
+ db.setBdbName(bdbName);
+ db.setLogPath(logPath);
+ BufferedReader r = new BufferedReader(
+ new InputStreamReader(System.in));
+ String line;
+ int exitCode = 0;
+ try {
+ db.init();
+ while((line = r.readLine()) != null) {
+ String parts[] = line.split(" ");
+ if(parts.length != 2) {
+ System.err.println("Bad input(" + line + ")");
+ System.exit(2);
+ }
+ db.addArcUrl(parts[0],parts[1]);
+ System.out.println("Added\t" + parts[0] + "\t" + parts[1]);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ exitCode = 1;
+ } catch (DatabaseException e) {
+ e.printStackTrace();
+ exitCode = 1;
+ } catch (ConfigurationException e) {
+ e.printStackTrace();
+ exitCode = 1;
+ } finally {
+ try {
+ db.shutdownDB();
+ } catch (DatabaseException e) {
+ e.printStackTrace();
+ exitCode = 1;
+ }
+ }
+ System.exit(exitCode);
+ }
+}
Copied: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileLocationDBClient.java (from rev 2210, trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/FileLocationDBClient.java)
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileLocationDBClient.java (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileLocationDBClient.java 2008-06-05 20:38:55 UTC (rev 2282)
@@ -0,0 +1,416 @@
+/* FileLocationDBClient
+ *
+ * $Id$
+ *
+ * Created on 5:59:49 PM Aug 21, 2006.
+ *
+ * Copyright (C) 2006 Internet Archive.
+ *
+ * This file is part of Wayback.
+ *
+ * Wayback is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * any later version.
+ *
+ * Wayback is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser Public License
+ * along with Wayback; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+package org.archive.wayback.resourcestore.locationdb;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.logging.Logger;
+
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.HttpStatus;
+import org.apache.commons.httpclient.NameValuePair;
+import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.commons.httpclient.methods.PostMethod;
+import org.apache.commons.httpclient.util.ParameterFormatter;
+import org.archive.wayback.resourcestore.locationdb.FileLocationDBServlet;
+
+/**
+ *
+ *
+ * @author brad
+ * @version $Date$, $Revision$
+ */
+public class FileLocationDBClient {
+ private static final Logger LOGGER = Logger.getLogger(FileLocationDBClient
+ .class.getName());
+
+ private final static String ARC_SUFFIX = ".arc";
+ private final static String ARC_GZ_SUFFIX = ".arc.gz";
+ private final static String WARC_SUFFIX = ".warc";
+ private final static String WARC_GZ_SUFFIX = ".warc.gz";
+ private final static String OK_RESPONSE_PREFIX = "OK ";
+ private HttpClient client = null;
+
+ private String serverUrl = null;
+
+ /**
+ * @param serverUrl
+ */
+ public FileLocationDBClient(final String serverUrl) {
+ super();
+ this.serverUrl = serverUrl;
+ this.client = new HttpClient();
+ }
+
+ /**
+ * @return long value representing the current end "mark" of the db log
+ * @throws NumberFormatException
+ * @throws IOException
+ */
+ public long getCurrentMark() throws NumberFormatException, IOException {
+ NameValuePair[] args = {
+ new NameValuePair(
+ FileLocationDBServlet.OPERATION_ARGUMENT,
+ FileLocationDBServlet.GETMARK_OPERATION),
+ };
+ return Long.parseLong(doGetMethod(args));
+ }
+
+ /**
+ * @param start
+ * @param end
+ * @return Iterator of arc file names between marks start and end
+ * @throws IOException
+ */
+ public Iterator<String> getArcsBetweenMarks(long start, long end)
+ throws IOException {
+ NameValuePair[] args = {
+ new NameValuePair(
+ FileLocationDBServlet.OPERATION_ARGUMENT,
+ FileLocationDBServlet.GETRANGE_OPERATION),
+ new NameValuePair(
+ FileLocationDBServlet.START_ARGUMENT,
+ String.valueOf(start)),
+ new NameValuePair(
+ FileLocationDBServlet.END_ARGUMENT,
+ String.valueOf(end))
+ };
+ return Arrays.asList(doGetMethod(args).split("\n")).iterator();
+ }
+
+ /**
+ * return an array of String URLs for all known locations of the ARC file
+ * in the DB.
+ * @param arcName
+ * @return String[] of URLs to arcName
+ * @throws IOException
+ */
+ public String[] arcToUrls(final String arcName) throws IOException {
+
+ NameValuePair[] args = {
+ new NameValuePair(
+ FileLocationDBServlet.OPERATION_ARGUMENT,
+ FileLocationDBServlet.LOOKUP_OPERATION),
+
+ new NameValuePair(
+ FileLocationDBServlet.NAME_ARGUMENT,
+ arcName)
+ };
+ String locations = doGetMethod(args);
+ if(locations != null) {
+ return locations.split("\n");
+ }
+ return null;
+ }
+
+
+ /**
+ * add an Url location for an arcName, unless it already exists
+ * @param arcName
+ * @param arcUrl
+ * @throws IOException
+ */
+ public void addArcUrl(final String arcName, final String arcUrl)
+ throws IOException {
+ doPostMethod(FileLocationDBServlet.ADD_OPERATION, arcName, arcUrl);
+ }
+
+ /**
+ * remove a single Url location for an arcName, if it exists
+ * @param arcName
+ * @param arcUrl
+ * @throws IOException
+ */
+ public void removeArcUrl(final String arcName, final String arcUrl)
+ throws IOException {
+ doPostMethod(FileLocationDBServlet.REMOVE_OPERATION, arcName, arcUrl);
+ }
+
+ private String doGetMethod(NameValuePair[] data) throws IOException {
+ ParameterFormatter formatter = new ParameterFormatter();
+ formatter.setAlwaysUseQuotes(false);
+ StringBuilder finalUrl = new StringBuilder(serverUrl);
+ if(data.length > 0) {
+ finalUrl.append("?");
+ }
+ for(int i = 0; i < data.length; i++) {
+ if(i == 0) {
+ finalUrl.append("?");
+ } else {
+ finalUrl.append("&");
+ }
+ finalUrl.append(formatter.format(data[i]));
+ }
+
+ GetMethod method = new GetMethod(finalUrl.toString());
+
+ int statusCode = client.executeMethod(method);
+ if (statusCode != HttpStatus.SC_OK) {
+ throw new IOException("Method failed: " + method.getStatusLine());
+ }
+ String responseString = method.getResponseBodyAsString();
+ if(!responseString.startsWith(OK_RESPONSE_PREFIX)) {
+ if(responseString.startsWith(FileLocationDBServlet.NO_LOCATION_PREFIX)) {
+ return null;
+ }
+ throw new IOException(responseString);
+ }
+ return responseString.substring(OK_RESPONSE_PREFIX.length()+1);
+ }
+
+ private void doPostMethod(final String operation, final String arcName,
+ final String arcUrl)
+ throws IOException {
+ PostMethod method = new PostMethod(serverUrl);
+ NameValuePair[] data = {
+ new NameValuePair(FileLocationDBServlet.OPERATION_ARGUMENT,
+ operation),
+ new NameValuePair(FileLocationDBServlet.NAME_ARGUMENT,
+ arcName),
+ new NameValuePair(FileLocationDBServlet.URL_ARGUMENT,
+ arcUrl)
+ };
+ method.setRequestBody(data);
+ int statusCode = client.executeMethod(method);
+ if (statusCode != HttpStatus.SC_OK) {
+ throw new IOException("Method failed: " + method.getStatusLine());
+ }
+ String responseString = method.getResponseBodyAsString();
+ if(!responseString.startsWith(OK_RESPONSE_PREFIX)) {
+ throw new IOException(responseString);
+ }
+ }
+
+ private static void USAGE(String message) {
+ System.err.print("USAGE: " + message + "\n" +
+ "\t[lookup|add|remove|sync] ...\n" +
+ "\n" +
+ "\t lookup LOCATION-DB-URL ARC\n" +
+ "\t\temit all known URLs for arc ARC\n" +
+ "\n" +
+ "\t add LOCATION-DB-URL ARC URL\n" +
+ "\t\tinform locationDB that ARC is located at URL\n" +
+ "\n" +
+ "\t remove LOCATION-DB-URL ARC URL\n" +
+ "\t\tremove reference to ARC at URL in locationDB\n" +
+ "\n" +
+ "\t sync LOCATION-DB-URL DIR DIR-URL\n" +
+ "\t\tscan directory DIR, and submit all ARC files therein\n" +
+ "\t\tto locationDB at url DIR-URL/ARC\n" +
+ "\n" +
+ "\t get-mark LOCATION-DB-URL\n" +
+ "\t\temit an identifier for the current marker in the \n" +
+ "\t\tlocationDB log. These identifiers can be used with the\n" +
+ "\t\tmark-range operation.\n" +
+ "\n" +
+ "\t mark-range LOCATION-DB-URL START END\n" +
+ "\t\temit to STDOUT one line with the name of all ARC files\n" +
+ "\t\tadded to the locationDB between marks START and END\n" +
+ "\n" +
+ "\t add-stream LOCATION-DB-URL\n" +
+ "\t\tread lines from STDIN formatted like:\n" +
+ "\t\t\tNAME<SPACE>URL\n" +
+ "\t\tand for each line, inform locationDB that file NAME is\n" +
+ "\t\tlocated at URL\n"
+ );
+ System.exit(2);
+ }
+
+ /**
+ * @param args
+ */
+ public static void main(String[] args) {
+ if(args.length < 2) {
+ USAGE("");
+ System.exit(1);
+ }
+ String operation = args[0];
+ String url = args[1];
+ if(!url.startsWith("http://")) {
+ USAGE("URL argument 1 must begin with http://");
+ }
+
+ FileLocationDBClient locationClient = new FileLocationDBClient(url);
+
+ if(operation.equalsIgnoreCase("add-stream")) {
+ BufferedReader r = new BufferedReader(
+ new InputStreamReader(System.in));
+ String line;
+ try {
+ while((line = r.readLine()) != null) {
+ String parts[] = line.split(" ");
+ if(parts.length != 2) {
+ System.err.println("Bad input(" + line + ")");
+ System.exit(2);
+ }
+ locationClient.addArcUrl(parts[0],parts[1]);
+ System.out.println("Added\t" + parts[0] + "\t" + parts[1]);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+
+ } else {
+ if(args.length < 3) {
+ USAGE("");
+ System.exit(1);
+ }
+ String arc = args[2];
+ if(operation.equalsIgnoreCase("lookup")) {
+ if(args.length < 3) {
+ USAGE("lookup LOCATION-URL ARC");
+ }
+ try {
+ String[] locations = locationClient.arcToUrls(arc);
+ if(locations == null) {
+ System.err.println("No locations for " + arc);
+ System.exit(1);
+ }
+ for(int i=0; i <locations.length; i++) {
+ System.out.println(locations[i]);
+ }
+ } catch (IOException e) {
+ System.err.println(e.getMessage());
+ System.exit(1);
+ }
+
+ } else if(operation.equalsIgnoreCase("get-mark")) {
+ if(args.length != 2) {
+ USAGE("get-mark LOCATION-URL");
+ }
+ try {
+ long mark = locationClient.getCurrentMark();
+ System.out.println(mark);
+ } catch (IOException e) {
+ System.err.println(e.getMessage());
+ System.exit(1);
+ }
+
+ } else if(operation.equalsIgnoreCase("mark-range")) {
+ if(args.length != 4) {
+ USAGE("mark-range LOCATION-URL START END");
+ }
+ long start = Long.parseLong(args[3]);
+ long end = Long.parseLong(args[4]);
+ try {
+ Iterator<String> it =
+ locationClient.getArcsBetweenMarks(start,end);
+ while(it.hasNext()) {
+ String next = (String) it.next();
+ System.out.println(next);
+ }
+ } catch (IOException e) {
+ System.err.println(e.getMessage());
+ System.exit(1);
+ }
+
+
+ } else if(operation.equalsIgnoreCase("add")) {
+ if(args.length != 4) {
+ USAGE("add LOCATION-URL ARC ARC-URL");
+ }
+ String arcUrl = args[3];
+ if(!arcUrl.startsWith("http://")) {
+ USAGE("ARC-URL argument 4 must begin with http://");
+ }
+ try {
+ locationClient.addArcUrl(arc,arcUrl);
+ System.out.println("OK");
+ } catch (IOException e) {
+ System.err.println(e.getMessage());
+ System.exit(1);
+ }
+
+ } else if(operation.equalsIgnoreCase("remove")) {
+
+ if(args.length != 4) {
+ USAGE("remove LOCATION-URL ARC ARC-URL");
+ }
+ String arcUrl = args[3];
+ if(!arcUrl.startsWith("http://")) {
+ USAGE("ARC-URL argument 4 must begin with http://");
+ }
+ try {
+ locationClient.removeArcUrl(arc,arcUrl);
+ System.out.println("OK");
+ } catch (IOException e) {
+ System.err.println(e.getMessage());
+ System.exit(1);
+ }
+
+ } else if(operation.equalsIgnoreCase("sync")) {
+
+ if(args.length != 4) {
+ USAGE("sync LOCATION-URL DIR DIR-URL");
+ }
+ File dir = new File(arc);
+ String dirUrl = args[3];
+ if(!dirUrl.startsWith("http://")) {
+ USAGE("DIR-URL argument 4 must begin with http://");
+ }
+ try {
+ if(!dir.isDirectory()) {
+ USAGE("DIR " + arc + " is not a directory");
+ }
+
+ FileFilter filter = new FileFilter() {
+ public boolean accept(File daFile) {
+ return daFile.isFile() &&
+ (daFile.getName().endsWith(ARC_SUFFIX) ||
+ daFile.getName().endsWith(ARC_GZ_SUFFIX) ||
+ daFile.getName().endsWith(WARC_SUFFIX) ||
+ daFile.getName().endsWith(WARC_GZ_SUFFIX));
+ }
+ };
+
+ File[] files = dir.listFiles(filter);
+ if(files == null) {
+ throw new IOException("Directory " + dir.getAbsolutePath() +
+ " is not a directory or had an IO error");
+ }
+ for(int i = 0; i < files.length; i++) {
+ File file = files[i];
+ String name = file.getName();
+ String fileUrl = dirUrl + name;
+ LOGGER.info("Adding location " + fileUrl + " for file " + name);
+ locationClient.addArcUrl(name,fileUrl);
+ }
+ } catch (IOException e) {
+ System.err.println(e.getMessage());
+ System.exit(1);
+ }
+
+ } else {
+ USAGE(" unknown operation " + operation);
+ }
+ }
+ }
+}
Copied: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileLocationDBLog.java (from rev 2055, trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/FileLocationDBLog.java)
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileLocationDBLog.java (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileLocationDBLog.java 2008-06-05 20:38:55 UTC (rev 2282)
@@ -0,0 +1,173 @@
+/* FileLocationDBLog
+ *
+ * $Id$
+ *
+ * Created on 2:38:18 PM Aug 18, 2006.
+ *
+ * Copyright (C) 2006 Internet Archive.
+ *
+ * This file is part of Wayback.
+ *
+ * Wayback is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * any later version.
+ *
+ * Wayback is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser Public License
+ * along with Wayback; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+package org.archive.wayback.resourcestore.locationdb;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+
+import org.archive.wayback.exception.ConfigurationException;
+import org.archive.wayback.util.CloseableIterator;
+import org.archive.wayback.util.flatfile.RecordIterator;
+
+import sun.reflect.generics.reflectiveObjects.NotImplementedException;
+
+/**
+ *
+ *
+ * @author brad
+ * @version $Date$, $Revision$
+ */
+public class FileLocationDBLog extends File {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = -9128222006544481378L;
+
+ /**
+ * @param pathname
+ * @throws ConfigurationException
+ */
+ public FileLocationDBLog(String pathname) throws ConfigurationException {
+ super(pathname);
+ if (!isFile()) {
+ if (exists()) {
+ throw new ConfigurationException("path(" + pathname
+ + ") exists but is not a file!");
+ }
+ try {
+ if (!createNewFile()) {
+ throw new ConfigurationException(
+ "Unable to create empty file " + pathname);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ throw new ConfigurationException("Unable to create empty file "
+ + pathname);
+ }
+ }
+ }
+
+ /**
+ * @return long value indicating the current end position of the log
+ */
+ public long getCurrentMark() {
+ return length();
+ }
+
+ /**
+ * @param start
+ * @param end
+ * @return CleanableIterator that returns all arcs between start and end
+ * @throws IOException
+ */
+ public CloseableIterator<String> getArcsBetweenMarks(long start, long end)
+ throws IOException {
+
+ RandomAccessFile raf = new RandomAccessFile(this, "r");
+ raf.seek(start);
+ BufferedReader is = new BufferedReader(new FileReader(raf.getFD()));
+ return new BufferedRangeIterator(new RecordIterator(is),end - start);
+ }
+
+ /**
+ * @param arcName
+ * @throws IOException
+ */
+ public synchronized void addArc(String arcName) throws IOException {
+ FileWriter writer = new FileWriter(this, true);
+ writer.write(arcName + "\n");
+ writer.flush();
+ writer.close();
+ }
+
+ private class BufferedRangeIterator implements CloseableIterator<String> {
+ private RecordIterator itr;
+ private long bytesToSend;
+ private long bytesSent;
+ private String next;
+ private boolean done;
+ /**
+ * @param itr
+ * @param bytesToSend
+ */
+ public BufferedRangeIterator(RecordIterator itr, long bytesToSend) {
+ this.itr = itr;
+ this.bytesToSend = bytesToSend;
+ bytesSent = 0;
+ next = null;
+ done = false;
+ }
+ /* (non-Javadoc)
+ * @see org.archive.wayback.util.CleanableIterator#clean()
+ */
+ public void close() throws IOException {
+ if(done == false) {
+ itr.close();
+ done = true;
+ }
+ }
+
+ /* (non-Javadoc)
+ * @see java.util.Iterator#hasNext()
+ */
+ public boolean hasNext() {
+ if(done) return false;
+ if(next != null) return true;
+ if((bytesSent >= bytesToSend) || !itr.hasNext()) {
+ try {
+ close();
+ } catch (IOException e) {
+ // TODO This is lame. What is the right way?
+ throw new RuntimeException(e);
+ }
+ return false;
+ }
+ next = (String) itr.next();
+ return true;
+ }
+
+ /* (non-Javadoc)
+ * @see java.util.Iterator#next()
+ */
+ public String next() {
+ String returnString = next;
+ next = null;
+ bytesSent += returnString.length() + 1; // TODO: not X-platform!
+ return returnString;
+ }
+
+ /* (non-Javadoc)
+ * @see java.util.Iterator#remove()
+ */
+ public void remove() {
+ throw new NotImplementedException();
+ }
+ }
+}
Copied: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileLocationDBServlet.java (from rev 2055, trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/FileLocationDBServlet.java)
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileLocationDBServlet.java (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileLocationDBServlet.java 2008-06-05 20:38:55 UTC (rev 2282)
@@ -0,0 +1,170 @@
+/* FileLocationDBServlet
+ *
+ * $Id$
+ *
+ * Created on 5:35:31 PM Aug 21, 2006.
+ *
+ * Copyright (C) 2006 Internet Archive.
+ *
+ * This file is part of Wayback.
+ *
+ * Wayback is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * any later version.
+ *
+ * Wayback is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser Public License
+ * along with Wayback; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+package org.archive.wayback.resourcestore.locationdb;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.text.ParseException;
+import java.util.Iterator;
+import java.util.Map;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.archive.wayback.resourcestore.locationdb.FileLocationDB;
+import org.archive.wayback.webapp.ServletRequestContext;
+
+import com.sleepycat.je.DatabaseException;
+
+/**
+ *
+ *
+ * @author brad
+ * @version $Date$, $Revision$
+ */
+public class FileLocationDBServlet extends ServletRequestContext {
+
+ protected static final String OPERATION_ARGUMENT = "operation";
+ protected static final String NAME_ARGUMENT = "name";
+ protected static final String URL_ARGUMENT = "url";
+ protected static final String START_ARGUMENT = "start";
+ protected static final String END_ARGUMENT = "end";
+ protected static final String LOOKUP_OPERATION = "lookup";
+ protected static final String GETMARK_OPERATION = "getmark";
+ protected static final String GETRANGE_OPERATION = "getrange";
+ protected static final String ADD_OPERATION = "add";
+ protected static final String REMOVE_OPERATION = "remove";
+ protected static final String NO_LOCATION_PREFIX = "ERROR No locations for";
+
+ private static final long serialVersionUID = 1L;
+ private FileLocationDB locationDB = null;
+
+ public boolean handleRequest(HttpServletRequest httpRequest,
+ HttpServletResponse httpResponse) throws IOException,
+ ServletException {
+ @SuppressWarnings("unchecked")
+ Map<String,String[]> queryMap = httpRequest.getParameterMap();
+ String message;
+ FileLocationDB locationDB = getLocationDB();
+ try {
+ message = handleOperation(locationDB,queryMap);
+ httpResponse.setStatus(HttpServletResponse.SC_OK);
+ httpResponse.setContentType("text/plain");
+ OutputStream os = httpResponse.getOutputStream();
+ os.write(message.getBytes());
+ } catch (ParseException e) {
+ e.printStackTrace();
+ httpResponse.sendError(HttpServletResponse.SC_BAD_REQUEST,
+ e.getMessage());
+ }
+ return true;
+ }
+
+ private String handleOperation(FileLocationDB locationDB,
+ Map<String,String[]> queryMap)
+ throws ParseException {
+
+ String operation = getRequiredMapParam(queryMap, OPERATION_ARGUMENT);
+ String message;
+ try {
+ if (operation.equals(LOOKUP_OPERATION)) {
+ String arcName = getRequiredMapParam(queryMap, NAME_ARGUMENT);
+
+ message = NO_LOCATION_PREFIX + " " + arcName;
+ String arcUrls[] = locationDB.arcToUrls(arcName);
+ if (arcUrls != null && arcUrls.length > 0) {
+ StringBuffer buf = new StringBuffer("OK ");
+ for (int i = 0; i < arcUrls.length; i++) {
+ buf.append("\n");
+ buf.append(arcUrls[i]);
+ }
+ message = buf.toString();
+ }
+
+ } else if (operation.equals(GETMARK_OPERATION)) {
+
+ message = "OK \n" + String.valueOf(locationDB.getCurrentMark());
+
+ } else if (operation.equals(GETRANGE_OPERATION)) {
+
+ long start = Long.parseLong(getRequiredMapParam(queryMap, START_ARGUMENT));
+ long end = Long.parseLong(getRequiredMapParam(queryMap, END_ARGUMENT));
+ Iterator<String> itr = locationDB.getArcsBetweenMarks(start,end);
+ StringBuilder str = new StringBuilder();
+ str.append("OK ");
+ while(itr.hasNext()) {
+ str.append("\n");
+ str.append((String)itr.next());
+ }
+ message = str.toString();
+
+ } else {
+
+ String arcName = getRequiredMapParam(queryMap, NAME_ARGUMENT);
+ String arcUrl = getRequiredMapParam(queryMap, URL_ARGUMENT);
+ if (operation.equals(ADD_OPERATION)) {
+
+ locationDB.addArcUrl(arcName, arcUrl);
+ message = "OK added url " + arcUrl + " for " + arcName;
+
+ } else if (operation.equals(REMOVE_OPERATION)) {
+
+ getLocationDB().removeArcUrl(arcName, arcUrl);
+ message = "OK removed url " + arcUrl + " for " + arcName;
+
+ } else {
+
+ throw new ParseException("Unknown operation. Must be one "
+ + "of " + LOOKUP_OPERATION + "," + ADD_OPERATION
+ + ", or " + REMOVE_OPERATION + ".", 0);
+ }
+ }
+
+ } catch (DatabaseException e) {
+ e.printStackTrace();
+ message = e.getMessage();
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ message = e.getMessage();
+ }
+ return message;
+ }
+
+ /**
+ * @return the locationDB
+ */
+ public FileLocationDB getLocationDB() {
+ return locationDB;
+ }
+
+ /**
+ * @param locationDB the locationDB to set
+ */
+ public void setLocationDB(FileLocationDB locationDB) {
+ this.locationDB = locationDB;
+ }
+}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
