Menu
▾
▴
archive-access-cvs
Revision: 2588
http://archive-access.svn.sourceforge.net/archive-access/?rev=2588&view=rev
Author: bradtofel
Date: 2008-08-29 22:18:41 +0000 (Fri, 29 Aug 2008)
Log Message:
-----------
FEATURE: now supports second offset specification form: ".../fileproxy/foo.arc.gz/12345" to return data starting at offset 12345.
FEATURE: now attempts to load from multiple locations if they are present in the locationDB
FEATURE: now proxies from local files as well as remote HTTP urls, including combinations of some local files, and some URLs for the same name.
Modified Paths:
--------------
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileProxyServlet.java
Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileProxyServlet.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileProxyServlet.java 2008-08-28 22:04:08 UTC (rev 2587)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileProxyServlet.java 2008-08-29 22:18:41 UTC (rev 2588)
@@ -24,11 +24,16 @@
*/
package org.archive.wayback.resourcestore.locationdb;
+import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.io.RandomAccessFile;
import java.net.URL;
import java.net.URLConnection;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
@@ -46,64 +51,145 @@
* @version $Date$, $Revision$
*/
public class FileProxyServlet extends ServletRequestContext {
+ private static final Logger LOGGER = Logger.getLogger(FileProxyServlet.class
+ .getName());
+ private static final int BUF_SIZE = 4096;
private static final String RANGE_HTTP_HEADER = "Range";
- private static final String CONTENT_TYPE_HEADER = "Content-Type";
- private static final String CONTENT_TYPE = "application/x-gzip";
- /**
- *
- */
+ private static final String DEFAULT_CONTENT_TYPE = "application/x-gzip";
+ private static final String HEADER_BYTES_PREFIX = "bytes=";
+ private static final String HEADER_BYTES_SUFFIX= "-";
+
+ private static final String FILE_REGEX = "/([^/]*)$";
+ private static final String FILE_OFFSET_REGEX = "/([^/]*)/(\\d*)$";
+
+ private static final Pattern FILE_PATTERN =
+ Pattern.compile(FILE_REGEX);
+ private static final Pattern FILE_OFFSET_PATTERN =
+ Pattern.compile(FILE_OFFSET_REGEX);
+
private static final long serialVersionUID = 1L;
+
private ResourceFileLocationDB locationDB = null;
public boolean handleRequest(HttpServletRequest httpRequest,
HttpServletResponse httpResponse) throws IOException,
ServletException {
- String name = httpRequest.getRequestURI();
- name = name.substring(name.lastIndexOf('/')+1);
- if(name.length() == 0) {
+ ResourceLocation location = parseRequest(httpRequest);
+ if(location == null) {
httpResponse.sendError(HttpServletResponse.SC_BAD_REQUEST,
- "no/invalid name");
+ "no/invalid name");
} else {
-
- String urls[] = locationDB.nameToUrls(name);
+ String urls[] = locationDB.nameToUrls(location.getName());
+
if(urls == null || urls.length == 0) {
+ LOGGER.warning("No locations for " + location.getName());
httpResponse.sendError(HttpServletResponse.SC_NOT_FOUND,
- "Unable to locate("+name+")");
+ "Unable to locate("+ location.getName() +")");
} else {
+
+ DataSource ds = null;
+ for(String urlString : urls) {
+ try {
+ ds = locationToDataSource(urlString,
+ location.getOffset());
+ if(ds != null) {
+ break;
+ }
+ } catch(IOException e) {
+ LOGGER.warning("failed proxy of " + urlString + " " +
+ e.getLocalizedMessage());
+ }
+ }
+ if(ds == null) {
+ LOGGER.warning("No successful locations for " +
+ location.getName());
+ httpResponse.sendError(HttpServletResponse.SC_BAD_GATEWAY,
+ "failed proxy of ("+ location.getName() +")");
+
+ } else {
+ httpResponse.setStatus(HttpServletResponse.SC_OK);
+ // BUGBUG: this will be broken for non compressed data...
+ httpResponse.setContentType(ds.getContentType());
+ ds.copyTo(httpResponse.getOutputStream());
+ }
+ }
+ }
+ return true;
+ }
- String urlString = urls[0];
- String rangeHeader = httpRequest.getHeader(RANGE_HTTP_HEADER);
- URL url = new URL(urlString);
- URLConnection conn = url.openConnection();
+ private DataSource locationToDataSource(String location, long offset)
+ throws IOException {
+ DataSource ds = null;
+ if(location.startsWith("http://")) {
+ URL url = new URL(location);
+ URLConnection conn = url.openConnection();
+ if(offset != 0) {
+ conn.addRequestProperty(RANGE_HTTP_HEADER,
+ HEADER_BYTES_PREFIX + String.valueOf(offset) +
+ HEADER_BYTES_SUFFIX);
+ }
+
+ ds = new URLDataSource(conn.getInputStream(),conn.getContentType());
+
+ } else {
+ // assume a local file path:
+ File f = new File(location);
+ if(f.isFile() && f.canRead()) {
+ long size = f.length();
+ if(size < offset) {
+ throw new IOException("short file " + location + " cannot" +
+ " seek to offset " + offset);
+ }
+ RandomAccessFile raf = new RandomAccessFile(f,"r");
+ raf.seek(offset);
+ // BUGBUG: is it compressed?
+ ds = new FileDataSource(raf,DEFAULT_CONTENT_TYPE);
+
+ } else {
+ throw new IOException("No readable file at " + location);
+ }
+
+ }
+
+ return ds;
+ }
+
+ private ResourceLocation parseRequest(HttpServletRequest request) {
+ ResourceLocation location = null;
+
+ String path = request.getRequestURI();
+ Matcher fo = FILE_OFFSET_PATTERN.matcher(path);
+ if(fo.find()) {
+ location = new ResourceLocation(fo.group(1),
+ Long.parseLong(fo.group(2)));
+ } else {
+ fo = FILE_PATTERN.matcher(path);
+ if(fo.find()) {
+ String rangeHeader = request.getHeader(RANGE_HTTP_HEADER);
if(rangeHeader != null) {
- conn.addRequestProperty(RANGE_HTTP_HEADER,rangeHeader);
- }
- InputStream is = conn.getInputStream();
- httpResponse.setStatus(HttpServletResponse.SC_OK);
- String typeHeader = conn.getHeaderField(CONTENT_TYPE_HEADER);
- if(typeHeader == null) {
- typeHeader = CONTENT_TYPE;
- }
- httpResponse.setContentType(typeHeader);
- OutputStream os = httpResponse.getOutputStream();
- int BUF_SIZE = 4096;
- byte[] buffer = new byte[BUF_SIZE];
- try {
- for(int r = -1; (r = is.read(buffer, 0, BUF_SIZE)) != -1;) {
- os.write(buffer, 0, r);
+ if(rangeHeader.startsWith(HEADER_BYTES_PREFIX)) {
+ rangeHeader = rangeHeader.substring(
+ HEADER_BYTES_PREFIX.length());
+ if(rangeHeader.endsWith(HEADER_BYTES_SUFFIX)) {
+ rangeHeader = rangeHeader.substring(0,
+ rangeHeader.length() -
+ HEADER_BYTES_SUFFIX.length());
+ }
}
- } finally {
- is.close();
+ location = new ResourceLocation(fo.group(1),
+ Long.parseLong(rangeHeader));
+ } else {
+ location = new ResourceLocation(fo.group(1));
}
}
}
- return true;
+ return location;
}
-
+
/**
* @return the locationDB
*/
@@ -117,4 +203,71 @@
public void setLocationDB(ResourceFileLocationDB locationDB) {
this.locationDB = locationDB;
}
+
+ private class ResourceLocation {
+ private String name = null;
+ private long offset = 0;
+ public ResourceLocation(String name, long offset) {
+ this.name = name;
+ this.offset = offset;
+ }
+ public ResourceLocation(String name) {
+ this(name,0);
+ }
+ public String getName() {
+ return name;
+ }
+ public long getOffset() {
+ return offset;
+ }
+ }
+
+ private interface DataSource {
+ public void copyTo(OutputStream os) throws IOException;
+ public String getContentType();
+ }
+ private class FileDataSource implements DataSource {
+ private RandomAccessFile raf = null;
+ private String contentType = null;
+ public FileDataSource(RandomAccessFile raf, String contentType) {
+ this.raf = raf;
+ this.contentType = contentType;
+ }
+ public String getContentType() {
+ return contentType;
+ }
+ public void copyTo(OutputStream os) throws IOException {
+ byte[] buffer = new byte[BUF_SIZE];
+ try {
+ int r = -1;
+ while((r = raf.read(buffer, 0, BUF_SIZE)) != -1) {
+ os.write(buffer, 0, r);
+ }
+ } finally {
+ raf.close();
+ }
+ }
+ }
+ private class URLDataSource implements DataSource {
+ private InputStream is = null;
+ private String contentType = null;
+ public URLDataSource(InputStream is,String contentType) {
+ this.is = is;
+ this.contentType = contentType;
+ }
+ public String getContentType() {
+ return contentType;
+ }
+ public void copyTo(OutputStream os) throws IOException {
+ byte[] buffer = new byte[BUF_SIZE];
+ try {
+ int r = -1;
+ while((r = is.read(buffer, 0, BUF_SIZE)) != -1) {
+ os.write(buffer, 0, r);
+ }
+ } finally {
+ is.close();
+ }
+ }
+ }
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
Revision: 2826
http://archive-access.svn.sourceforge.net/archive-access/?rev=2826&view=rev
Author: bradtofel
Date: 2009-10-23 00:44:47 +0000 (Fri, 23 Oct 2009)
Log Message:
-----------
FEATURE: Now handles all the HTTP communications explicitly, as I couldn't find a small HttpClient lib that enabled fine grained control over the various socket operations that can timeout. So, now you can configure specific timeouts.
Modified Paths:
--------------
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileProxyServlet.java
Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileProxyServlet.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileProxyServlet.java 2009-10-23 00:40:41 UTC (rev 2825)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/locationdb/FileProxyServlet.java 2009-10-23 00:44:47 UTC (rev 2826)
@@ -29,8 +29,10 @@
import java.io.InputStream;
import java.io.OutputStream;
import java.io.RandomAccessFile;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.Socket;
import java.net.URL;
-import java.net.URLConnection;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -39,6 +41,10 @@
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import org.apache.commons.httpclient.ChunkedInputStream;
+import org.archive.util.anvl.ANVLRecord;
+import org.archive.wayback.util.http.HttpRequestMessage;
+import org.archive.wayback.util.http.HttpResponse;
import org.archive.wayback.webapp.ServletRequestContext;
/**
@@ -71,6 +77,10 @@
private static final long serialVersionUID = 1L;
private ResourceFileLocationDB locationDB = null;
+
+ private int socketTimeoutMs = 5000;
+
+ private int connectTimeoutMs = 1000;
public boolean handleRequest(HttpServletRequest httpRequest,
HttpServletResponse httpResponse) throws IOException,
@@ -114,6 +124,7 @@
httpResponse.setStatus(HttpServletResponse.SC_OK);
// BUGBUG: this will be broken for non compressed data...
httpResponse.setContentType(ds.getContentType());
+ httpResponse.setBufferSize(BUF_SIZE);
ds.copyTo(httpResponse.getOutputStream());
}
}
@@ -126,15 +137,51 @@
DataSource ds = null;
if(location.startsWith("http://")) {
URL url = new URL(location);
- URLConnection conn = url.openConnection();
+ String hostname = url.getHost();
+ int port = url.getPort();
+ if(port == -1) {
+ port = 80;
+ }
+ byte GET[] = "GET".getBytes();
+ byte HTTP11[] = "HTTP/1.1".getBytes();
+ InetAddress addr = InetAddress.getByName(hostname);
+ HttpRequestMessage requestMessage = new HttpRequestMessage(
+ GET,url.getFile().getBytes(),HTTP11);
+ ANVLRecord headers = new ANVLRecord();
+ headers.addLabelValue("Host", hostname);
+
+
if(offset != 0) {
- conn.addRequestProperty(RANGE_HTTP_HEADER,
+ headers.addLabelValue(RANGE_HTTP_HEADER,
HEADER_BYTES_PREFIX + String.valueOf(offset) +
- HEADER_BYTES_SUFFIX);
+ HEADER_BYTES_SUFFIX);
}
+ InetSocketAddress sockAddr = new InetSocketAddress(addr,port);
+ Socket socket = new Socket();
+ socket.setSoTimeout(socketTimeoutMs);
+ socket.setReceiveBufferSize(BUF_SIZE);
- ds = new URLDataSource(conn.getInputStream(),conn.getContentType());
+ socket.connect(sockAddr, connectTimeoutMs);
+ OutputStream socketOut = socket.getOutputStream();
+ InputStream socketIn = socket.getInputStream();
+ socketOut.write(requestMessage.getBytes(true));
+ socketOut.write(headers.getUTF8Bytes());
+ socketOut.flush();
+ HttpResponse response = HttpResponse.load(socketIn);
+ String contentType = response.getHeaders().asMap().get("Content-Type");
+ if(contentType == null) {
+ contentType = "application/unknown";
+ }
+ String xferEncoding = response.getHeaders().asMap().get("Transfer-Encoding");
+ if(xferEncoding != null) {
+ if(xferEncoding.equals("chunked")) {
+ socketIn = new ChunkedInputStream(socketIn);
+ }
+ }
+
+ ds = new URLDataSource(socketIn,contentType);
+
} else {
// assume a local file path:
File f = new File(location);
@@ -204,6 +251,34 @@
this.locationDB = locationDB;
}
+ /**
+ * @return the socketTimeoutMs
+ */
+ public int getSocketTimeoutMs() {
+ return socketTimeoutMs;
+ }
+
+ /**
+ * @param socketTimeoutMs the socketTimeoutMs to set
+ */
+ public void setSocketTimeoutMs(int socketTimeoutMs) {
+ this.socketTimeoutMs = socketTimeoutMs;
+ }
+
+ /**
+ * @return the connectTimeoutMs
+ */
+ public int getConnectTimeoutMs() {
+ return connectTimeoutMs;
+ }
+
+ /**
+ * @param connectTimeoutMs the connectTimeoutMs to set
+ */
+ public void setConnectTimeoutMs(int connectTimeoutMs) {
+ this.connectTimeoutMs = connectTimeoutMs;
+ }
+
private class ResourceLocation {
private String name = null;
private long offset = 0;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
×
Want the latest updates on software, tech news, and AI?
Get latest updates about software, tech news, and AI from SourceForge directly in your inbox once a month.
Thanks for helping keep SourceForge clean.
X