Menu
▾
▴
[Archive-access-cvs] SF.net SVN: archive-access: [2283] trunk/archive-access/projects/wayback/ wayback-core/src/main/java/org/archive/wayback/resourcestore/http
|
From: <bra...@us...> - 2008-06-05 20:39:07
|
Revision: 2283
http://archive-access.svn.sourceforge.net/archive-access/?rev=2283&view=rev
Author: bradtofel
Date: 2008-06-05 13:39:13 -0700 (Thu, 05 Jun 2008)
Log Message:
-----------
EMPTIED
Removed Paths:
-------------
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/ArcProxyServlet.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/FileLocationDB.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/FileLocationDBClient.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/FileLocationDBLog.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/FileLocationDBServlet.java
Deleted: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/ArcProxyServlet.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/ArcProxyServlet.java 2008-06-05 20:38:55 UTC (rev 2282)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/ArcProxyServlet.java 2008-06-05 20:39:13 UTC (rev 2283)
@@ -1,124 +0,0 @@
-/* ArcProxyServlet
- *
- * $Id$
- *
- * Created on 6:19:54 PM Aug 10, 2006.
- *
- * Copyright (C) 2006 Internet Archive.
- *
- * This file is part of Wayback.
- *
- * Wayback is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Lesser Public License as published by
- * the Free Software Foundation; either version 2.1 of the License, or
- * any later version.
- *
- * Wayback is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Lesser Public License for more details.
- *
- * You should have received a copy of the GNU Lesser Public License
- * along with Wayback; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
- */
-package org.archive.wayback.resourcestore.http;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URL;
-import java.net.URLConnection;
-import java.text.ParseException;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.archive.wayback.resourcestore.http.FileLocationDB;
-import org.archive.wayback.webapp.ServletRequestContext;
-
-import com.sleepycat.je.DatabaseException;
-
-/**
- *
- *
- * @author brad
- * @version $Date$, $Revision$
- */
-public class ArcProxyServlet extends ServletRequestContext {
-
- private static final String RANGE_HTTP_HEADER = "Range";
- private static final String CONTENT_TYPE_HEADER = "Content-Type";
- private static final String CONTENT_TYPE = "application/x-gzip";
- /**
- *
- */
- private static final long serialVersionUID = 1L;
- private FileLocationDB locationDB = null;
-
- public boolean handleRequest(HttpServletRequest httpRequest,
- HttpServletResponse httpResponse) throws IOException,
- ServletException {
-
- try {
- String arc = httpRequest.getRequestURI();
- arc = arc.substring(arc.lastIndexOf('/')+1);
- if(arc.length() == 0) {
- throw new ParseException("no/invalid arc",0);
- }
- String urls[] = locationDB.arcToUrls(arc);
- if(urls == null || urls.length == 0) {
- throw new DatabaseException("Unable to locate("+arc+")");
- }
- String urlString = urls[0];
- String rangeHeader = httpRequest.getHeader(RANGE_HTTP_HEADER);
- URL url = new URL(urlString);
- URLConnection conn = url.openConnection();
- if(rangeHeader != null) {
- conn.addRequestProperty(RANGE_HTTP_HEADER,rangeHeader);
- }
- InputStream is = conn.getInputStream();
- httpResponse.setStatus(HttpServletResponse.SC_OK);
- String typeHeader = conn.getHeaderField(CONTENT_TYPE_HEADER);
- if(typeHeader == null) {
- typeHeader = CONTENT_TYPE;
- }
- httpResponse.setContentType(typeHeader);
- OutputStream os = httpResponse.getOutputStream();
- int BUF_SIZE = 4096;
- byte[] buffer = new byte[BUF_SIZE];
- try {
- for (int r = -1; (r = is.read(buffer, 0, BUF_SIZE)) != -1;) {
- os.write(buffer, 0, r);
- }
- } finally {
- is.close();
- }
- } catch (ParseException e) {
- e.printStackTrace();
- httpResponse.sendError(HttpServletResponse.SC_BAD_REQUEST,
- e.getMessage());
- } catch (DatabaseException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- httpResponse.sendError(HttpServletResponse.SC_NOT_FOUND,
- e.getMessage());
- }
- return true;
- }
-
- /**
- * @return the locationDB
- */
- public FileLocationDB getLocationDB() {
- return locationDB;
- }
-
- /**
- * @param locationDB the locationDB to set
- */
- public void setLocationDB(FileLocationDB locationDB) {
- this.locationDB = locationDB;
- }
-}
Deleted: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/FileLocationDB.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/FileLocationDB.java 2008-06-05 20:38:55 UTC (rev 2282)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/FileLocationDB.java 2008-06-05 20:39:13 UTC (rev 2283)
@@ -1,305 +0,0 @@
-/* FileLocationDB
- *
- * $Id$
- *
- * Created on 3:08:59 PM Aug 18, 2006.
- *
- * Copyright (C) 2006 Internet Archive.
- *
- * This file is part of Wayback.
- *
- * Wayback is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Lesser Public License as published by
- * the Free Software Foundation; either version 2.1 of the License, or
- * any later version.
- *
- * Wayback is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Lesser Public License for more details.
- *
- * You should have received a copy of the GNU Lesser Public License
- * along with Wayback; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
- */
-package org.archive.wayback.resourcestore.http;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-
-import org.archive.wayback.bdb.BDBRecordSet;
-import org.archive.wayback.exception.ConfigurationException;
-import org.archive.wayback.util.CloseableIterator;
-
-import com.sleepycat.je.DatabaseException;
-
-/**
- *
- *
- * @author brad
- * @version $Date$, $Revision$
- */
-public class FileLocationDB extends BDBRecordSet {
-
- /**
- * String id for implementation class of FileLocationDBs.
- */
- public static final String FILE_LOCATION_DB_CLASS = "filelocationdb";
-
- protected static final String ARC_DB_PATH = "filelocationdb.path";
-
- protected static final String ARC_DB_NAME = "filelocationdb.name";
-
- protected static final String ARC_DB_LOG = "filelocationdb.logpath";
-
- private final static String urlDelimiter = " ";
-
- private final static String urlDelimiterRE = " ";
-
- private FileLocationDBLog log;
- private String logPath = null;
- private String bdbPath = null;
- private String bdbName = null;
-
- /**
- * Constructor
- */
- public FileLocationDB() {
- super();
- }
-
- /**
- * @throws DatabaseException
- * @throws ConfigurationException
- */
- public void init() throws DatabaseException, ConfigurationException {
- if(logPath == null) {
- throw new ConfigurationException("No logPath");
- }
- log = new FileLocationDBLog(logPath);
- initializeDB(bdbPath,bdbName);
- }
-
- /**
- * return an array of String URLs for all known locations of the ARC file
- * in the DB.
- * @param arcName
- * @return String[] of URLs to arcName
- * @throws DatabaseException
- */
- public String[] arcToUrls(final String arcName) throws DatabaseException {
-
- String[] arcUrls = null;
- String valueString = get(arcName);
- if(valueString != null && valueString.length() > 0) {
- arcUrls = valueString.split(urlDelimiterRE);
- }
- return arcUrls;
- }
-
- /**
- * add an Url location for an arcName, unless it already exists
- * @param arcName
- * @param arcUrl
- * @throws DatabaseException
- * @throws IOException
- */
- public void addArcUrl(final String arcName, final String arcUrl) throws DatabaseException, IOException {
-
- // need to first see if there is already an entry for this arcName.
- // if not, add arcUrl as the value.
- // if so, check the current arcUrl locations for arcName
- // if arcUrl exists, do nothing
- // if arcUrl does not exist, add, and set that as the value.
-
- String newValue = null;
- String oldValue = get(arcName);
- if(oldValue != null && oldValue.length() > 0) {
- String curUrls[] = oldValue.split(urlDelimiterRE);
- boolean found = false;
- for(int i=0; i < curUrls.length; i++) {
- if(arcUrl.equals(curUrls[i])) {
- found = true;
- break;
- }
- }
- if(found == false) {
- newValue = oldValue + " " + arcUrl;
- }
- } else {
- // null or empty value
- newValue = arcUrl;
- if(oldValue == null) log.addArc(arcName);
- }
-
- // did we find a value?
- if(newValue != null) {
- put(arcName,newValue);
- }
- }
-
- /**
- * remove a single Url location for an arcName, if it exists
- * @param arcName
- * @param arcUrl
- * @throws DatabaseException
- */
- public void removeArcUrl(final String arcName, final String arcUrl) throws DatabaseException {
- // need to first see if there is already an entry for this arcName.
- // if not, do nothing
- // if so, loop thru all current arcUrl locations for arcName
- // keep any that are not arcUrl
- // if any locations are left, update to the new value, sans arcUrl
- // if none are left, remove the entry from the db
-
- StringBuilder newValue = new StringBuilder();
- String oldValue = get(arcName);
- if(oldValue != null && oldValue.length() > 0) {
- String curUrls[] = oldValue.split(urlDelimiterRE);
-
- for(int i=0; i < curUrls.length; i++) {
- if(!arcUrl.equals(curUrls[i])) {
- if(newValue.length() > 0) {
- newValue.append(urlDelimiter);
- }
- newValue.append(curUrls[i]);
- }
- }
-
- if(newValue.length() > 0) {
-
- // update
- put(arcName, newValue.toString());
-
- } else {
-
- // remove the entry:
- delete(arcName);
- }
- }
- }
-
- /**
- * @param start
- * @param end
- * @return Iterator for traversing arcs between start and end.
- * @throws IOException
- */
- public CloseableIterator<String> getArcsBetweenMarks(long start, long end)
- throws IOException {
- return log.getArcsBetweenMarks(start, end);
- }
-
- /**
- * @return current "Mark" for the log. Currently, it's just the length of
- * the log file.
- */
- public long getCurrentMark() {
- return log.getCurrentMark();
- }
-
- /**
- * @return the logPath
- */
- public String getLogPath() {
- return logPath;
- }
-
- /**
- * @param logPath the logPath to set
- */
- public void setLogPath(String logPath) {
- this.logPath = logPath;
- }
-
- /**
- * @return the bdbPath
- */
- public String getBdbPath() {
- return bdbPath;
- }
-
- /**
- * @param bdbPath the bdbPath to set
- */
- public void setBdbPath(String bdbPath) {
- this.bdbPath = bdbPath;
- }
-
- /**
- * @return the bdbName
- */
- public String getBdbName() {
- return bdbName;
- }
-
- /**
- * @param bdbName the bdbName to set
- */
- public void setBdbName(String bdbName) {
- this.bdbName = bdbName;
- }
- private static void USAGE(String message) {
- System.err.print("USAGE: " + message + "\n" +
- "\tDBDIR DBNAME LOGPATH\n" +
- "\n" +
- "\t\tread lines from STDIN formatted like:\n" +
- "\t\t\tNAME<SPACE>URL\n" +
- "\t\tand for each line, add to locationDB that file NAME is\n" +
- "\t\tlocated at URL. Use locationDB in DBDIR at DBNAME, \n" +
- "\t\tcreating if it does not exist.\n"
- );
- System.exit(2);
- }
-
- /**
- * @param args
- */
- public static void main(String[] args) {
- if(args.length != 3) {
- USAGE("");
- System.exit(1);
- }
- String bdbPath = args[0];
- String bdbName = args[1];
- String logPath = args[2];
- FileLocationDB db = new FileLocationDB();
- db.setBdbPath(bdbPath);
- db.setBdbName(bdbName);
- db.setLogPath(logPath);
- BufferedReader r = new BufferedReader(
- new InputStreamReader(System.in));
- String line;
- int exitCode = 0;
- try {
- db.init();
- while((line = r.readLine()) != null) {
- String parts[] = line.split(" ");
- if(parts.length != 2) {
- System.err.println("Bad input(" + line + ")");
- System.exit(2);
- }
- db.addArcUrl(parts[0],parts[1]);
- System.out.println("Added\t" + parts[0] + "\t" + parts[1]);
- }
- } catch (IOException e) {
- e.printStackTrace();
- exitCode = 1;
- } catch (DatabaseException e) {
- e.printStackTrace();
- exitCode = 1;
- } catch (ConfigurationException e) {
- e.printStackTrace();
- exitCode = 1;
- } finally {
- try {
- db.shutdownDB();
- } catch (DatabaseException e) {
- e.printStackTrace();
- exitCode = 1;
- }
- }
- System.exit(exitCode);
- }
-}
Deleted: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/FileLocationDBClient.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/FileLocationDBClient.java 2008-06-05 20:38:55 UTC (rev 2282)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/FileLocationDBClient.java 2008-06-05 20:39:13 UTC (rev 2283)
@@ -1,416 +0,0 @@
-/* FileLocationDBClient
- *
- * $Id$
- *
- * Created on 5:59:49 PM Aug 21, 2006.
- *
- * Copyright (C) 2006 Internet Archive.
- *
- * This file is part of Wayback.
- *
- * Wayback is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Lesser Public License as published by
- * the Free Software Foundation; either version 2.1 of the License, or
- * any later version.
- *
- * Wayback is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Lesser Public License for more details.
- *
- * You should have received a copy of the GNU Lesser Public License
- * along with Wayback; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
- */
-package org.archive.wayback.resourcestore.http;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileFilter;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.logging.Logger;
-
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpStatus;
-import org.apache.commons.httpclient.NameValuePair;
-import org.apache.commons.httpclient.methods.GetMethod;
-import org.apache.commons.httpclient.methods.PostMethod;
-import org.apache.commons.httpclient.util.ParameterFormatter;
-import org.archive.wayback.resourcestore.http.FileLocationDBServlet;
-
-/**
- *
- *
- * @author brad
- * @version $Date$, $Revision$
- */
-public class FileLocationDBClient {
- private static final Logger LOGGER = Logger.getLogger(FileLocationDBClient
- .class.getName());
-
- private final static String ARC_SUFFIX = ".arc";
- private final static String ARC_GZ_SUFFIX = ".arc.gz";
- private final static String WARC_SUFFIX = ".warc";
- private final static String WARC_GZ_SUFFIX = ".warc.gz";
- private final static String OK_RESPONSE_PREFIX = "OK ";
- private HttpClient client = null;
-
- private String serverUrl = null;
-
- /**
- * @param serverUrl
- */
- public FileLocationDBClient(final String serverUrl) {
- super();
- this.serverUrl = serverUrl;
- this.client = new HttpClient();
- }
-
- /**
- * @return long value representing the current end "mark" of the db log
- * @throws NumberFormatException
- * @throws IOException
- */
- public long getCurrentMark() throws NumberFormatException, IOException {
- NameValuePair[] args = {
- new NameValuePair(
- FileLocationDBServlet.OPERATION_ARGUMENT,
- FileLocationDBServlet.GETMARK_OPERATION),
- };
- return Long.parseLong(doGetMethod(args));
- }
-
- /**
- * @param start
- * @param end
- * @return Iterator of arc file names between marks start and end
- * @throws IOException
- */
- public Iterator<String> getArcsBetweenMarks(long start, long end)
- throws IOException {
- NameValuePair[] args = {
- new NameValuePair(
- FileLocationDBServlet.OPERATION_ARGUMENT,
- FileLocationDBServlet.GETRANGE_OPERATION),
- new NameValuePair(
- FileLocationDBServlet.START_ARGUMENT,
- String.valueOf(start)),
- new NameValuePair(
- FileLocationDBServlet.END_ARGUMENT,
- String.valueOf(end))
- };
- return Arrays.asList(doGetMethod(args).split("\n")).iterator();
- }
-
- /**
- * return an array of String URLs for all known locations of the ARC file
- * in the DB.
- * @param arcName
- * @return String[] of URLs to arcName
- * @throws IOException
- */
- public String[] arcToUrls(final String arcName) throws IOException {
-
- NameValuePair[] args = {
- new NameValuePair(
- FileLocationDBServlet.OPERATION_ARGUMENT,
- FileLocationDBServlet.LOOKUP_OPERATION),
-
- new NameValuePair(
- FileLocationDBServlet.NAME_ARGUMENT,
- arcName)
- };
- String locations = doGetMethod(args);
- if(locations != null) {
- return locations.split("\n");
- }
- return null;
- }
-
-
- /**
- * add an Url location for an arcName, unless it already exists
- * @param arcName
- * @param arcUrl
- * @throws IOException
- */
- public void addArcUrl(final String arcName, final String arcUrl)
- throws IOException {
- doPostMethod(FileLocationDBServlet.ADD_OPERATION, arcName, arcUrl);
- }
-
- /**
- * remove a single Url location for an arcName, if it exists
- * @param arcName
- * @param arcUrl
- * @throws IOException
- */
- public void removeArcUrl(final String arcName, final String arcUrl)
- throws IOException {
- doPostMethod(FileLocationDBServlet.REMOVE_OPERATION, arcName, arcUrl);
- }
-
- private String doGetMethod(NameValuePair[] data) throws IOException {
- ParameterFormatter formatter = new ParameterFormatter();
- formatter.setAlwaysUseQuotes(false);
- StringBuilder finalUrl = new StringBuilder(serverUrl);
- if(data.length > 0) {
- finalUrl.append("?");
- }
- for(int i = 0; i < data.length; i++) {
- if(i == 0) {
- finalUrl.append("?");
- } else {
- finalUrl.append("&");
- }
- finalUrl.append(formatter.format(data[i]));
- }
-
- GetMethod method = new GetMethod(finalUrl.toString());
-
- int statusCode = client.executeMethod(method);
- if (statusCode != HttpStatus.SC_OK) {
- throw new IOException("Method failed: " + method.getStatusLine());
- }
- String responseString = method.getResponseBodyAsString();
- if(!responseString.startsWith(OK_RESPONSE_PREFIX)) {
- if(responseString.startsWith(FileLocationDBServlet.NO_LOCATION_PREFIX)) {
- return null;
- }
- throw new IOException(responseString);
- }
- return responseString.substring(OK_RESPONSE_PREFIX.length()+1);
- }
-
- private void doPostMethod(final String operation, final String arcName,
- final String arcUrl)
- throws IOException {
- PostMethod method = new PostMethod(serverUrl);
- NameValuePair[] data = {
- new NameValuePair(FileLocationDBServlet.OPERATION_ARGUMENT,
- operation),
- new NameValuePair(FileLocationDBServlet.NAME_ARGUMENT,
- arcName),
- new NameValuePair(FileLocationDBServlet.URL_ARGUMENT,
- arcUrl)
- };
- method.setRequestBody(data);
- int statusCode = client.executeMethod(method);
- if (statusCode != HttpStatus.SC_OK) {
- throw new IOException("Method failed: " + method.getStatusLine());
- }
- String responseString = method.getResponseBodyAsString();
- if(!responseString.startsWith(OK_RESPONSE_PREFIX)) {
- throw new IOException(responseString);
- }
- }
-
- private static void USAGE(String message) {
- System.err.print("USAGE: " + message + "\n" +
- "\t[lookup|add|remove|sync] ...\n" +
- "\n" +
- "\t lookup LOCATION-DB-URL ARC\n" +
- "\t\temit all known URLs for arc ARC\n" +
- "\n" +
- "\t add LOCATION-DB-URL ARC URL\n" +
- "\t\tinform locationDB that ARC is located at URL\n" +
- "\n" +
- "\t remove LOCATION-DB-URL ARC URL\n" +
- "\t\tremove reference to ARC at URL in locationDB\n" +
- "\n" +
- "\t sync LOCATION-DB-URL DIR DIR-URL\n" +
- "\t\tscan directory DIR, and submit all ARC files therein\n" +
- "\t\tto locationDB at url DIR-URL/ARC\n" +
- "\n" +
- "\t get-mark LOCATION-DB-URL\n" +
- "\t\temit an identifier for the current marker in the \n" +
- "\t\tlocationDB log. These identifiers can be used with the\n" +
- "\t\tmark-range operation.\n" +
- "\n" +
- "\t mark-range LOCATION-DB-URL START END\n" +
- "\t\temit to STDOUT one line with the name of all ARC files\n" +
- "\t\tadded to the locationDB between marks START and END\n" +
- "\n" +
- "\t add-stream LOCATION-DB-URL\n" +
- "\t\tread lines from STDIN formatted like:\n" +
- "\t\t\tNAME<SPACE>URL\n" +
- "\t\tand for each line, inform locationDB that file NAME is\n" +
- "\t\tlocated at URL\n"
- );
- System.exit(2);
- }
-
- /**
- * @param args
- */
- public static void main(String[] args) {
- if(args.length < 2) {
- USAGE("");
- System.exit(1);
- }
- String operation = args[0];
- String url = args[1];
- if(!url.startsWith("http://")) {
- USAGE("URL argument 1 must begin with http://");
- }
-
- FileLocationDBClient locationClient = new FileLocationDBClient(url);
-
- if(operation.equalsIgnoreCase("add-stream")) {
- BufferedReader r = new BufferedReader(
- new InputStreamReader(System.in));
- String line;
- try {
- while((line = r.readLine()) != null) {
- String parts[] = line.split(" ");
- if(parts.length != 2) {
- System.err.println("Bad input(" + line + ")");
- System.exit(2);
- }
- locationClient.addArcUrl(parts[0],parts[1]);
- System.out.println("Added\t" + parts[0] + "\t" + parts[1]);
- }
- } catch (IOException e) {
- e.printStackTrace();
- System.exit(1);
- }
-
- } else {
- if(args.length < 3) {
- USAGE("");
- System.exit(1);
- }
- String arc = args[2];
- if(operation.equalsIgnoreCase("lookup")) {
- if(args.length < 3) {
- USAGE("lookup LOCATION-URL ARC");
- }
- try {
- String[] locations = locationClient.arcToUrls(arc);
- if(locations == null) {
- System.err.println("No locations for " + arc);
- System.exit(1);
- }
- for(int i=0; i <locations.length; i++) {
- System.out.println(locations[i]);
- }
- } catch (IOException e) {
- System.err.println(e.getMessage());
- System.exit(1);
- }
-
- } else if(operation.equalsIgnoreCase("get-mark")) {
- if(args.length != 2) {
- USAGE("get-mark LOCATION-URL");
- }
- try {
- long mark = locationClient.getCurrentMark();
- System.out.println(mark);
- } catch (IOException e) {
- System.err.println(e.getMessage());
- System.exit(1);
- }
-
- } else if(operation.equalsIgnoreCase("mark-range")) {
- if(args.length != 4) {
- USAGE("mark-range LOCATION-URL START END");
- }
- long start = Long.parseLong(args[3]);
- long end = Long.parseLong(args[4]);
- try {
- Iterator<String> it =
- locationClient.getArcsBetweenMarks(start,end);
- while(it.hasNext()) {
- String next = (String) it.next();
- System.out.println(next);
- }
- } catch (IOException e) {
- System.err.println(e.getMessage());
- System.exit(1);
- }
-
-
- } else if(operation.equalsIgnoreCase("add")) {
- if(args.length != 4) {
- USAGE("add LOCATION-URL ARC ARC-URL");
- }
- String arcUrl = args[3];
- if(!arcUrl.startsWith("http://")) {
- USAGE("ARC-URL argument 4 must begin with http://");
- }
- try {
- locationClient.addArcUrl(arc,arcUrl);
- System.out.println("OK");
- } catch (IOException e) {
- System.err.println(e.getMessage());
- System.exit(1);
- }
-
- } else if(operation.equalsIgnoreCase("remove")) {
-
- if(args.length != 4) {
- USAGE("remove LOCATION-URL ARC ARC-URL");
- }
- String arcUrl = args[3];
- if(!arcUrl.startsWith("http://")) {
- USAGE("ARC-URL argument 4 must begin with http://");
- }
- try {
- locationClient.removeArcUrl(arc,arcUrl);
- System.out.println("OK");
- } catch (IOException e) {
- System.err.println(e.getMessage());
- System.exit(1);
- }
-
- } else if(operation.equalsIgnoreCase("sync")) {
-
- if(args.length != 4) {
- USAGE("sync LOCATION-URL DIR DIR-URL");
- }
- File dir = new File(arc);
- String dirUrl = args[3];
- if(!dirUrl.startsWith("http://")) {
- USAGE("DIR-URL argument 4 must begin with http://");
- }
- try {
- if(!dir.isDirectory()) {
- USAGE("DIR " + arc + " is not a directory");
- }
-
- FileFilter filter = new FileFilter() {
- public boolean accept(File daFile) {
- return daFile.isFile() &&
- (daFile.getName().endsWith(ARC_SUFFIX) ||
- daFile.getName().endsWith(ARC_GZ_SUFFIX) ||
- daFile.getName().endsWith(WARC_SUFFIX) ||
- daFile.getName().endsWith(WARC_GZ_SUFFIX));
- }
- };
-
- File[] files = dir.listFiles(filter);
- if(files == null) {
- throw new IOException("Directory " + dir.getAbsolutePath() +
- " is not a directory or had an IO error");
- }
- for(int i = 0; i < files.length; i++) {
- File file = files[i];
- String name = file.getName();
- String fileUrl = dirUrl + name;
- LOGGER.info("Adding location " + fileUrl + " for file " + name);
- locationClient.addArcUrl(name,fileUrl);
- }
- } catch (IOException e) {
- System.err.println(e.getMessage());
- System.exit(1);
- }
-
- } else {
- USAGE(" unknown operation " + operation);
- }
- }
- }
-}
Deleted: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/FileLocationDBLog.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/FileLocationDBLog.java 2008-06-05 20:38:55 UTC (rev 2282)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/FileLocationDBLog.java 2008-06-05 20:39:13 UTC (rev 2283)
@@ -1,173 +0,0 @@
-/* FileLocationDBLog
- *
- * $Id$
- *
- * Created on 2:38:18 PM Aug 18, 2006.
- *
- * Copyright (C) 2006 Internet Archive.
- *
- * This file is part of Wayback.
- *
- * Wayback is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Lesser Public License as published by
- * the Free Software Foundation; either version 2.1 of the License, or
- * any later version.
- *
- * Wayback is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Lesser Public License for more details.
- *
- * You should have received a copy of the GNU Lesser Public License
- * along with Wayback; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
- */
-package org.archive.wayback.resourcestore.http;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-
-import org.archive.wayback.exception.ConfigurationException;
-import org.archive.wayback.util.CloseableIterator;
-import org.archive.wayback.util.flatfile.RecordIterator;
-
-import sun.reflect.generics.reflectiveObjects.NotImplementedException;
-
-/**
- *
- *
- * @author brad
- * @version $Date$, $Revision$
- */
-public class FileLocationDBLog extends File {
-
- /**
- *
- */
- private static final long serialVersionUID = -9128222006544481378L;
-
- /**
- * @param pathname
- * @throws ConfigurationException
- */
- public FileLocationDBLog(String pathname) throws ConfigurationException {
- super(pathname);
- if (!isFile()) {
- if (exists()) {
- throw new ConfigurationException("path(" + pathname
- + ") exists but is not a file!");
- }
- try {
- if (!createNewFile()) {
- throw new ConfigurationException(
- "Unable to create empty file " + pathname);
- }
- } catch (IOException e) {
- e.printStackTrace();
- throw new ConfigurationException("Unable to create empty file "
- + pathname);
- }
- }
- }
-
- /**
- * @return long value indicating the current end position of the log
- */
- public long getCurrentMark() {
- return length();
- }
-
- /**
- * @param start
- * @param end
- * @return CleanableIterator that returns all arcs between start and end
- * @throws IOException
- */
- public CloseableIterator<String> getArcsBetweenMarks(long start, long end)
- throws IOException {
-
- RandomAccessFile raf = new RandomAccessFile(this, "r");
- raf.seek(start);
- BufferedReader is = new BufferedReader(new FileReader(raf.getFD()));
- return new BufferedRangeIterator(new RecordIterator(is),end - start);
- }
-
- /**
- * @param arcName
- * @throws IOException
- */
- public synchronized void addArc(String arcName) throws IOException {
- FileWriter writer = new FileWriter(this, true);
- writer.write(arcName + "\n");
- writer.flush();
- writer.close();
- }
-
- private class BufferedRangeIterator implements CloseableIterator<String> {
- private RecordIterator itr;
- private long bytesToSend;
- private long bytesSent;
- private String next;
- private boolean done;
- /**
- * @param itr
- * @param bytesToSend
- */
- public BufferedRangeIterator(RecordIterator itr, long bytesToSend) {
- this.itr = itr;
- this.bytesToSend = bytesToSend;
- bytesSent = 0;
- next = null;
- done = false;
- }
- /* (non-Javadoc)
- * @see org.archive.wayback.util.CleanableIterator#clean()
- */
- public void close() throws IOException {
- if(done == false) {
- itr.close();
- done = true;
- }
- }
-
- /* (non-Javadoc)
- * @see java.util.Iterator#hasNext()
- */
- public boolean hasNext() {
- if(done) return false;
- if(next != null) return true;
- if((bytesSent >= bytesToSend) || !itr.hasNext()) {
- try {
- close();
- } catch (IOException e) {
- // TODO This is lame. What is the right way?
- throw new RuntimeException(e);
- }
- return false;
- }
- next = (String) itr.next();
- return true;
- }
-
- /* (non-Javadoc)
- * @see java.util.Iterator#next()
- */
- public String next() {
- String returnString = next;
- next = null;
- bytesSent += returnString.length() + 1; // TODO: not X-platform!
- return returnString;
- }
-
- /* (non-Javadoc)
- * @see java.util.Iterator#remove()
- */
- public void remove() {
- throw new NotImplementedException();
- }
- }
-}
Deleted: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/FileLocationDBServlet.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/FileLocationDBServlet.java 2008-06-05 20:38:55 UTC (rev 2282)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/http/FileLocationDBServlet.java 2008-06-05 20:39:13 UTC (rev 2283)
@@ -1,170 +0,0 @@
-/* FileLocationDBServlet
- *
- * $Id$
- *
- * Created on 5:35:31 PM Aug 21, 2006.
- *
- * Copyright (C) 2006 Internet Archive.
- *
- * This file is part of Wayback.
- *
- * Wayback is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Lesser Public License as published by
- * the Free Software Foundation; either version 2.1 of the License, or
- * any later version.
- *
- * Wayback is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Lesser Public License for more details.
- *
- * You should have received a copy of the GNU Lesser Public License
- * along with Wayback; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
- */
-package org.archive.wayback.resourcestore.http;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.text.ParseException;
-import java.util.Iterator;
-import java.util.Map;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.archive.wayback.resourcestore.http.FileLocationDB;
-import org.archive.wayback.webapp.ServletRequestContext;
-
-import com.sleepycat.je.DatabaseException;
-
-/**
- *
- *
- * @author brad
- * @version $Date$, $Revision$
- */
-public class FileLocationDBServlet extends ServletRequestContext {
-
- protected static final String OPERATION_ARGUMENT = "operation";
- protected static final String NAME_ARGUMENT = "name";
- protected static final String URL_ARGUMENT = "url";
- protected static final String START_ARGUMENT = "start";
- protected static final String END_ARGUMENT = "end";
- protected static final String LOOKUP_OPERATION = "lookup";
- protected static final String GETMARK_OPERATION = "getmark";
- protected static final String GETRANGE_OPERATION = "getrange";
- protected static final String ADD_OPERATION = "add";
- protected static final String REMOVE_OPERATION = "remove";
- protected static final String NO_LOCATION_PREFIX = "ERROR No locations for";
-
- private static final long serialVersionUID = 1L;
- private FileLocationDB locationDB = null;
-
- public boolean handleRequest(HttpServletRequest httpRequest,
- HttpServletResponse httpResponse) throws IOException,
- ServletException {
- @SuppressWarnings("unchecked")
- Map<String,String[]> queryMap = httpRequest.getParameterMap();
- String message;
- FileLocationDB locationDB = getLocationDB();
- try {
- message = handleOperation(locationDB,queryMap);
- httpResponse.setStatus(HttpServletResponse.SC_OK);
- httpResponse.setContentType("text/plain");
- OutputStream os = httpResponse.getOutputStream();
- os.write(message.getBytes());
- } catch (ParseException e) {
- e.printStackTrace();
- httpResponse.sendError(HttpServletResponse.SC_BAD_REQUEST,
- e.getMessage());
- }
- return true;
- }
-
- private String handleOperation(FileLocationDB locationDB,
- Map<String,String[]> queryMap)
- throws ParseException {
-
- String operation = getRequiredMapParam(queryMap, OPERATION_ARGUMENT);
- String message;
- try {
- if (operation.equals(LOOKUP_OPERATION)) {
- String arcName = getRequiredMapParam(queryMap, NAME_ARGUMENT);
-
- message = NO_LOCATION_PREFIX + " " + arcName;
- String arcUrls[] = locationDB.arcToUrls(arcName);
- if (arcUrls != null && arcUrls.length > 0) {
- StringBuffer buf = new StringBuffer("OK ");
- for (int i = 0; i < arcUrls.length; i++) {
- buf.append("\n");
- buf.append(arcUrls[i]);
- }
- message = buf.toString();
- }
-
- } else if (operation.equals(GETMARK_OPERATION)) {
-
- message = "OK \n" + String.valueOf(locationDB.getCurrentMark());
-
- } else if (operation.equals(GETRANGE_OPERATION)) {
-
- long start = Long.parseLong(getRequiredMapParam(queryMap, START_ARGUMENT));
- long end = Long.parseLong(getRequiredMapParam(queryMap, END_ARGUMENT));
- Iterator<String> itr = locationDB.getArcsBetweenMarks(start,end);
- StringBuilder str = new StringBuilder();
- str.append("OK ");
- while(itr.hasNext()) {
- str.append("\n");
- str.append((String)itr.next());
- }
- message = str.toString();
-
- } else {
-
- String arcName = getRequiredMapParam(queryMap, NAME_ARGUMENT);
- String arcUrl = getRequiredMapParam(queryMap, URL_ARGUMENT);
- if (operation.equals(ADD_OPERATION)) {
-
- locationDB.addArcUrl(arcName, arcUrl);
- message = "OK added url " + arcUrl + " for " + arcName;
-
- } else if (operation.equals(REMOVE_OPERATION)) {
-
- getLocationDB().removeArcUrl(arcName, arcUrl);
- message = "OK removed url " + arcUrl + " for " + arcName;
-
- } else {
-
- throw new ParseException("Unknown operation. Must be one "
- + "of " + LOOKUP_OPERATION + "," + ADD_OPERATION
- + ", or " + REMOVE_OPERATION + ".", 0);
- }
- }
-
- } catch (DatabaseException e) {
- e.printStackTrace();
- message = e.getMessage();
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- message = e.getMessage();
- }
- return message;
- }
-
- /**
- * @return the locationDB
- */
- public FileLocationDB getLocationDB() {
- return locationDB;
- }
-
- /**
- * @param locationDB the locationDB to set
- */
- public void setLocationDB(FileLocationDB locationDB) {
- this.locationDB = locationDB;
- }
-}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
