Menu
▾
▴
[Archive-access-cvs] SF.net SVN: archive-access: [2322] trunk/archive-access/projects/wayback/ wayback-core/src/main/java/org/archive/wayback/resourcestore
|
From: <bra...@us...> - 2008-06-25 01:30:51
|
Revision: 2322
http://archive-access.svn.sourceforge.net/archive-access/?rev=2322&view=rev
Author: bradtofel
Date: 2008-06-24 18:30:59 -0700 (Tue, 24 Jun 2008)
Log Message:
-----------
REFACTOR: moved ARC/WARC record to Resource code into resourcefile package.
Added Paths:
-----------
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/resourcefile/ArcResource.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/resourcefile/ResourceFactory.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/resourcefile/WarcResource.java
Removed Paths:
-------------
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/ArcResource.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/ResourceFactory.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/WarcResource.java
Deleted: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/ArcResource.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/ArcResource.java 2008-06-25 01:30:18 UTC (rev 2321)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/ArcResource.java 2008-06-25 01:30:59 UTC (rev 2322)
@@ -1,170 +0,0 @@
-package org.archive.wayback.resourcestore;
-
-import java.io.IOException;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.Hashtable;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-import java.util.logging.Logger;
-
-import org.apache.commons.httpclient.Header;
-import org.archive.io.ArchiveRecord;
-import org.archive.io.arc.ARCReader;
-import org.archive.io.arc.ARCRecord;
-import org.archive.wayback.core.Resource;
-
-public class ArcResource extends Resource {
- /**
- * Logger for this class
- */
- private static final Logger LOGGER = Logger.getLogger(ArcResource.class
- .getName());
-
- /**
- * String prefix for ARC file related metadata namespace of keys within
- * metaData Properties bag.
- */
- private static String ARC_META_PREFIX = "arcmeta.";
- /**
- * String prefix for HTTP Header related metadata namespace of keys within
- * metaData Properties bag.
- */
- private static String HTTP_HEADER_PREFIX = "httpheader.";
- /**
- * object for ARCRecord
- */
- ARCRecord arcRecord = null;
- /**
- * object for ARCReader -- need to hold on to this in order to call close()
- * to release filehandle after completing access to this record. optional
- */
- ARCReader arcReader = null;
- /**
- * flag to indicate if the ARCRecord skipHTTPHeader() has been called
- */
- boolean parsedHeader = false;
- /**
- * Expandable property bag for holding metadata associated with this
- * resource
- */
- Hashtable<String,String> metaData = new Hashtable<String,String>();
-
- /**
- * Constructor
- *
- * @param rec
- * @param reader
- */
- public ArcResource(final ARCRecord rec,final ARCReader reader) {
- super();
- arcRecord = rec;
- arcReader = reader;
- setInputStream(rec);
- }
-
- /** parse the headers on the underlying ARC record, and extract all
- * @throws IOException
- */
- public void parseHeaders () throws IOException {
- if(!parsedHeader) {
- arcRecord.skipHttpHeader();
- // copy all HTTP headers to metaData, prefixing with
- // HTTP_HEADER_PREFIX
- Header[] headers = arcRecord.getHttpHeaders();
- if (headers != null) {
- for (int i = 0; i < headers.length; i++) {
- String value = headers[i].getValue();
- String name = headers[i].getName();
- metaData.put(HTTP_HEADER_PREFIX + name,value);
- }
- }
-
- // copy all ARC record header fields to metaData, prefixing with
- // ARC_META_PREFIX
- @SuppressWarnings("unchecked")
- Map<String,Object> headerMetaMap = arcRecord.getMetaData().getHeaderFields();
- Set<String> keys = headerMetaMap.keySet();
- Iterator<String> itr = keys.iterator();
- while(itr.hasNext()) {
- String metaKey = itr.next();
- Object value = headerMetaMap.get(metaKey);
- String metaValue = "";
- if(value != null) {
- metaValue = value.toString();
- }
- metaData.put(ARC_META_PREFIX + metaKey,metaValue);
- }
-
- parsedHeader = true;
- }
- }
-
- /**
- * @param prefix
- * @return a Properties of all elements in metaData starting with 'prefix'.
- * keys in the returned Properties have 'prefix' removed.
- */
- public Map<String,String> filterMeta(String prefix) {
- HashMap<String,String> matching = new HashMap<String,String>();
- for (Enumeration<String> e = metaData.keys(); e.hasMoreElements();) {
- String key = e.nextElement();
- if (key.startsWith(prefix)) {
- String finalKey = key.substring(prefix.length());
- String value = metaData.get(key);
- matching.put(finalKey, value);
- }
- }
- return matching;
- }
-
- /**
- * @return a Properties containing all HTTP header fields for this record
- */
- public Map<String,String> getHttpHeaders() {
- return filterMeta(HTTP_HEADER_PREFIX);
- }
-
- /**
- * @return a Properties containing all ARC Meta fields for this record
- */
- public Map<String,String> getARCMetadata() {
- return filterMeta(ARC_META_PREFIX);
- }
-
- /**
- * (non-Javadoc)
- * @see org.archive.io.arc.ARCRecord#getStatusCode()
- * @return int HTTP status code returned with this document.
- */
- public int getStatusCode() {
- return arcRecord.getStatusCode();
- }
-
- /**
- * @return the ARCRecord underlying this Resource.
- */
- public ArchiveRecord getArcRecord() {
- return arcRecord;
- }
-
- /* (non-Javadoc)
- * @see org.archive.io.arc.ARCRecord#close()
- */
- public void close() throws IOException {
- //LOGGER.info("About to close..("+arcReader+")");
- arcRecord.close();
- if(arcReader != null) {
- arcReader.close();
- LOGGER.info("closed..("+arcReader+")");
- }
- }
-
- /**
- * @return byte length claimed in ARC record metadata line.
- */
- public long getRecordLength() {
- return arcRecord.getMetaData().getLength();
- }
-}
Deleted: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/ResourceFactory.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/ResourceFactory.java 2008-06-25 01:30:18 UTC (rev 2321)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/ResourceFactory.java 2008-06-25 01:30:59 UTC (rev 2322)
@@ -1,105 +0,0 @@
-package org.archive.wayback.resourcestore;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URL;
-
-import org.archive.io.ArchiveRecord;
-import org.archive.io.arc.ARCReader;
-import org.archive.io.arc.ARCReaderFactory;
-import org.archive.io.arc.ARCRecord;
-import org.archive.io.warc.WARCReader;
-import org.archive.io.warc.WARCReaderFactory;
-import org.archive.io.warc.WARCRecord;
-import org.archive.wayback.core.Resource;
-import org.archive.wayback.exception.ResourceNotAvailableException;
-
-/**
- * Static factory class for constructing ARC/WARC Resources from
- * File/URL + offset.
- *
- * @author brad
- * @version $Date$, $Revision$
- */
-public class ResourceFactory {
-
- public static Resource getResource(File file, long offset)
- throws IOException, ResourceNotAvailableException {
-
- Resource r = null;
- String name = file.getName();
- if (name.endsWith(LocalResourceStore.OPEN_EXTENSION)) {
- name = name.substring(0, name.length()
- - LocalResourceStore.OPEN_EXTENSION.length());
- }
- if (isArc(name)) {
-
- ARCReader reader = ARCReaderFactory.get(file,offset);
- r = ARCArchiveRecordToResource(reader.get(),reader);
-
- } else if (isWarc(name)) {
-
- WARCReader reader = WARCReaderFactory.get(file,offset);
- r = WARCArchiveRecordToResource(reader.get(),reader);
-
- } else {
- throw new ResourceNotAvailableException("Unknown extension");
- }
-
- return r;
- }
-
- public static Resource getResource(URL url, long offset)
- throws IOException, ResourceNotAvailableException {
- Resource r = null;
- String name = url.getFile();
- if (isArc(name)) {
-
- ARCReader reader = ARCReaderFactory.get(url, offset);
- r = ARCArchiveRecordToResource(reader.get(),reader);
-
- } else if (isWarc(name)) {
-
- WARCReader reader = WARCReaderFactory.get(url, offset);
- r = WARCArchiveRecordToResource(reader.get(),reader);
-
- } else {
- throw new ResourceNotAvailableException("Unknown extension");
- }
- return r;
- }
-
- private static boolean isArc(final String name) {
-
- return (name.endsWith(LocalResourceStore.ARC_EXTENSION)
- || name.endsWith(LocalResourceStore.ARC_GZ_EXTENSION));
- }
-
- private static boolean isWarc(final String name) {
-
- return (name.endsWith(LocalResourceStore.WARC_EXTENSION)
- || name.endsWith(LocalResourceStore.WARC_GZ_EXTENSION));
- }
-
- private static Resource ARCArchiveRecordToResource(ArchiveRecord rec,
- ARCReader reader) throws ResourceNotAvailableException, IOException {
-
- if (!(rec instanceof ARCRecord)) {
- throw new ResourceNotAvailableException("Bad ARCRecord format");
- }
- ArcResource ar = new ArcResource((ARCRecord) rec, reader);
- ar.parseHeaders();
- return ar;
- }
-
- private static Resource WARCArchiveRecordToResource(ArchiveRecord rec,
- WARCReader reader) throws ResourceNotAvailableException, IOException {
-
- if (!(rec instanceof WARCRecord)) {
- throw new ResourceNotAvailableException("Bad WARCRecord format");
- }
- WarcResource wr = new WarcResource((WARCRecord) rec, reader);
- wr.parseHeaders();
- return wr;
- }
-}
Deleted: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/WarcResource.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/WarcResource.java 2008-06-25 01:30:18 UTC (rev 2321)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/WarcResource.java 2008-06-25 01:30:59 UTC (rev 2322)
@@ -1,98 +0,0 @@
-package org.archive.wayback.resourcestore;
-
-import java.io.IOException;
-import java.util.Hashtable;
-import java.util.Map;
-
-import org.apache.commons.httpclient.Header;
-import org.apache.commons.httpclient.HttpParser;
-import org.apache.commons.httpclient.StatusLine;
-import org.apache.commons.httpclient.util.EncodingUtil;
-import org.archive.io.RecoverableIOException;
-import org.archive.io.arc.ARCConstants;
-import org.archive.io.warc.WARCReader;
-import org.archive.io.warc.WARCRecord;
-import org.archive.wayback.core.Resource;
-
-public class WarcResource extends Resource {
- private WARCRecord rec = null;
- private WARCReader reader = null;
- private Map<String, String> headers = null;
- private long length = 0;
- private int status = 0;
- private boolean parsedHeaders = false;
- public WarcResource(WARCRecord rec, WARCReader reader) {
- this.rec = rec;
- this.reader = reader;
- }
-
- /**
- * @param bytes Array of bytes to examine for an EOL.
- * @return Count of end-of-line characters or zero if none.
- */
- private int getEolCharsCount(byte [] bytes) {
- int count = 0;
- if (bytes != null && bytes.length >=1 &&
- bytes[bytes.length - 1] == '\n') {
- count++;
- if (bytes.length >=2 && bytes[bytes.length -2] == '\r') {
- count++;
- }
- }
- return count;
- }
-
- public void parseHeaders() throws IOException {
- if(parsedHeaders) {
- return;
- }
-
- byte [] statusBytes = HttpParser.readRawLine(rec);
- int eolCharCount = getEolCharsCount(statusBytes);
- if (eolCharCount <= 0) {
- throw new RecoverableIOException("Failed to read http status where one " +
- " was expected: " + new String(statusBytes));
- }
- String statusLineStr = EncodingUtil.getString(statusBytes, 0,
- statusBytes.length - eolCharCount, ARCConstants.DEFAULT_ENCODING);
- if ((statusLineStr == null) ||
- !StatusLine.startsWithHTTP(statusLineStr)) {
- throw new RecoverableIOException("Failed parse of http status line.");
- }
- StatusLine statusLine = new StatusLine(statusLineStr);
-
- this.status = statusLine.getStatusCode();
-
- Header[] tmpHeaders = HttpParser.parseHeaders(rec,
- ARCConstants.DEFAULT_ENCODING);
- headers = new Hashtable<String,String>();
- for(Header header: tmpHeaders) {
- headers.put(header.getName(), header.getValue());
- }
- this.setInputStream(rec);
- parsedHeaders = true;
- }
-
-
- @Override
- public Map<String, String> getHttpHeaders() {
- return headers;
- }
-
- @Override
- public long getRecordLength() {
- // TODO Auto-generated method stub
- return length;
- }
-
- @Override
- public int getStatusCode() {
- return status;
- }
-
- @Override
- public void close() throws IOException {
- rec.close();
- reader.close();
- }
-}
Copied: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/resourcefile/ArcResource.java (from rev 2082, trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/ArcResource.java)
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/resourcefile/ArcResource.java (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/resourcefile/ArcResource.java 2008-06-25 01:30:59 UTC (rev 2322)
@@ -0,0 +1,170 @@
+package org.archive.wayback.resourcestore.resourcefile;
+
+import java.io.IOException;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Hashtable;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Logger;
+
+import org.apache.commons.httpclient.Header;
+import org.archive.io.ArchiveRecord;
+import org.archive.io.arc.ARCReader;
+import org.archive.io.arc.ARCRecord;
+import org.archive.wayback.core.Resource;
+
+public class ArcResource extends Resource {
+ /**
+ * Logger for this class
+ */
+ private static final Logger LOGGER = Logger.getLogger(ArcResource.class
+ .getName());
+
+ /**
+ * String prefix for ARC file related metadata namespace of keys within
+ * metaData Properties bag.
+ */
+ private static String ARC_META_PREFIX = "arcmeta.";
+ /**
+ * String prefix for HTTP Header related metadata namespace of keys within
+ * metaData Properties bag.
+ */
+ private static String HTTP_HEADER_PREFIX = "httpheader.";
+ /**
+ * object for ARCRecord
+ */
+ ARCRecord arcRecord = null;
+ /**
+ * object for ARCReader -- need to hold on to this in order to call close()
+ * to release filehandle after completing access to this record. optional
+ */
+ ARCReader arcReader = null;
+ /**
+ * flag to indicate if the ARCRecord skipHTTPHeader() has been called
+ */
+ boolean parsedHeader = false;
+ /**
+ * Expandable property bag for holding metadata associated with this
+ * resource
+ */
+ Hashtable<String,String> metaData = new Hashtable<String,String>();
+
+ /**
+ * Constructor
+ *
+ * @param rec
+ * @param reader
+ */
+ public ArcResource(final ARCRecord rec,final ARCReader reader) {
+ super();
+ arcRecord = rec;
+ arcReader = reader;
+ setInputStream(rec);
+ }
+
+ /** parse the headers on the underlying ARC record, and extract all
+ * @throws IOException
+ */
+ public void parseHeaders () throws IOException {
+ if(!parsedHeader) {
+ arcRecord.skipHttpHeader();
+ // copy all HTTP headers to metaData, prefixing with
+ // HTTP_HEADER_PREFIX
+ Header[] headers = arcRecord.getHttpHeaders();
+ if (headers != null) {
+ for (int i = 0; i < headers.length; i++) {
+ String value = headers[i].getValue();
+ String name = headers[i].getName();
+ metaData.put(HTTP_HEADER_PREFIX + name,value);
+ }
+ }
+
+ // copy all ARC record header fields to metaData, prefixing with
+ // ARC_META_PREFIX
+ @SuppressWarnings("unchecked")
+ Map<String,Object> headerMetaMap = arcRecord.getMetaData().getHeaderFields();
+ Set<String> keys = headerMetaMap.keySet();
+ Iterator<String> itr = keys.iterator();
+ while(itr.hasNext()) {
+ String metaKey = itr.next();
+ Object value = headerMetaMap.get(metaKey);
+ String metaValue = "";
+ if(value != null) {
+ metaValue = value.toString();
+ }
+ metaData.put(ARC_META_PREFIX + metaKey,metaValue);
+ }
+
+ parsedHeader = true;
+ }
+ }
+
+ /**
+ * @param prefix
+ * @return a Properties of all elements in metaData starting with 'prefix'.
+ * keys in the returned Properties have 'prefix' removed.
+ */
+ public Map<String,String> filterMeta(String prefix) {
+ HashMap<String,String> matching = new HashMap<String,String>();
+ for (Enumeration<String> e = metaData.keys(); e.hasMoreElements();) {
+ String key = e.nextElement();
+ if (key.startsWith(prefix)) {
+ String finalKey = key.substring(prefix.length());
+ String value = metaData.get(key);
+ matching.put(finalKey, value);
+ }
+ }
+ return matching;
+ }
+
+ /**
+ * @return a Properties containing all HTTP header fields for this record
+ */
+ public Map<String,String> getHttpHeaders() {
+ return filterMeta(HTTP_HEADER_PREFIX);
+ }
+
+ /**
+ * @return a Properties containing all ARC Meta fields for this record
+ */
+ public Map<String,String> getARCMetadata() {
+ return filterMeta(ARC_META_PREFIX);
+ }
+
+ /**
+ * (non-Javadoc)
+ * @see org.archive.io.arc.ARCRecord#getStatusCode()
+ * @return int HTTP status code returned with this document.
+ */
+ public int getStatusCode() {
+ return arcRecord.getStatusCode();
+ }
+
+ /**
+ * @return the ARCRecord underlying this Resource.
+ */
+ public ArchiveRecord getArcRecord() {
+ return arcRecord;
+ }
+
+ /* (non-Javadoc)
+ * @see org.archive.io.arc.ARCRecord#close()
+ */
+ public void close() throws IOException {
+ //LOGGER.info("About to close..("+arcReader+")");
+ arcRecord.close();
+ if(arcReader != null) {
+ arcReader.close();
+ LOGGER.info("closed..("+arcReader+")");
+ }
+ }
+
+ /**
+ * @return byte length claimed in ARC record metadata line.
+ */
+ public long getRecordLength() {
+ return arcRecord.getMetaData().getLength();
+ }
+}
Copied: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/resourcefile/ResourceFactory.java (from rev 2122, trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/ResourceFactory.java)
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/resourcefile/ResourceFactory.java (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/resourcefile/ResourceFactory.java 2008-06-25 01:30:59 UTC (rev 2322)
@@ -0,0 +1,105 @@
+package org.archive.wayback.resourcestore.resourcefile;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+
+import org.archive.io.ArchiveRecord;
+import org.archive.io.arc.ARCReader;
+import org.archive.io.arc.ARCReaderFactory;
+import org.archive.io.arc.ARCRecord;
+import org.archive.io.warc.WARCReader;
+import org.archive.io.warc.WARCReaderFactory;
+import org.archive.io.warc.WARCRecord;
+import org.archive.wayback.core.Resource;
+import org.archive.wayback.exception.ResourceNotAvailableException;
+
+/**
+ * Static factory class for constructing ARC/WARC Resources from
+ * File/URL + offset.
+ *
+ * @author brad
+ * @version $Date$, $Revision$
+ */
+public class ResourceFactory {
+
+ public static Resource getResource(File file, long offset)
+ throws IOException, ResourceNotAvailableException {
+
+ Resource r = null;
+ String name = file.getName();
+ if (name.endsWith(ArcWarcFilenameFilter.OPEN_SUFFIX)) {
+ name = name.substring(0, name.length()
+ - ArcWarcFilenameFilter.OPEN_SUFFIX.length());
+ }
+ if (isArc(name)) {
+
+ ARCReader reader = ARCReaderFactory.get(file,offset);
+ r = ARCArchiveRecordToResource(reader.get(),reader);
+
+ } else if (isWarc(name)) {
+
+ WARCReader reader = WARCReaderFactory.get(file,offset);
+ r = WARCArchiveRecordToResource(reader.get(),reader);
+
+ } else {
+ throw new ResourceNotAvailableException("Unknown extension");
+ }
+
+ return r;
+ }
+
+ public static Resource getResource(URL url, long offset)
+ throws IOException, ResourceNotAvailableException {
+ Resource r = null;
+ String name = url.getFile();
+ if (isArc(name)) {
+
+ ARCReader reader = ARCReaderFactory.get(url, offset);
+ r = ARCArchiveRecordToResource(reader.get(),reader);
+
+ } else if (isWarc(name)) {
+
+ WARCReader reader = WARCReaderFactory.get(url, offset);
+ r = WARCArchiveRecordToResource(reader.get(),reader);
+
+ } else {
+ throw new ResourceNotAvailableException("Unknown extension");
+ }
+ return r;
+ }
+
+ private static boolean isArc(final String name) {
+
+ return (name.endsWith(ArcWarcFilenameFilter.ARC_SUFFIX)
+ || name.endsWith(ArcWarcFilenameFilter.ARC_GZ_SUFFIX));
+ }
+
+ private static boolean isWarc(final String name) {
+
+ return (name.endsWith(ArcWarcFilenameFilter.WARC_SUFFIX)
+ || name.endsWith(ArcWarcFilenameFilter.WARC_GZ_SUFFIX));
+ }
+
+ private static Resource ARCArchiveRecordToResource(ArchiveRecord rec,
+ ARCReader reader) throws ResourceNotAvailableException, IOException {
+
+ if (!(rec instanceof ARCRecord)) {
+ throw new ResourceNotAvailableException("Bad ARCRecord format");
+ }
+ ArcResource ar = new ArcResource((ARCRecord) rec, reader);
+ ar.parseHeaders();
+ return ar;
+ }
+
+ private static Resource WARCArchiveRecordToResource(ArchiveRecord rec,
+ WARCReader reader) throws ResourceNotAvailableException, IOException {
+
+ if (!(rec instanceof WARCRecord)) {
+ throw new ResourceNotAvailableException("Bad WARCRecord format");
+ }
+ WarcResource wr = new WarcResource((WARCRecord) rec, reader);
+ wr.parseHeaders();
+ return wr;
+ }
+}
Copied: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/resourcefile/WarcResource.java (from rev 2082, trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/WarcResource.java)
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/resourcefile/WarcResource.java (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/resourcefile/WarcResource.java 2008-06-25 01:30:59 UTC (rev 2322)
@@ -0,0 +1,98 @@
+package org.archive.wayback.resourcestore.resourcefile;
+
+import java.io.IOException;
+import java.util.Hashtable;
+import java.util.Map;
+
+import org.apache.commons.httpclient.Header;
+import org.apache.commons.httpclient.HttpParser;
+import org.apache.commons.httpclient.StatusLine;
+import org.apache.commons.httpclient.util.EncodingUtil;
+import org.archive.io.RecoverableIOException;
+import org.archive.io.arc.ARCConstants;
+import org.archive.io.warc.WARCReader;
+import org.archive.io.warc.WARCRecord;
+import org.archive.wayback.core.Resource;
+
+public class WarcResource extends Resource {
+ private WARCRecord rec = null;
+ private WARCReader reader = null;
+ private Map<String, String> headers = null;
+ private long length = 0;
+ private int status = 0;
+ private boolean parsedHeaders = false;
+ public WarcResource(WARCRecord rec, WARCReader reader) {
+ this.rec = rec;
+ this.reader = reader;
+ }
+
+ /**
+ * @param bytes Array of bytes to examine for an EOL.
+ * @return Count of end-of-line characters or zero if none.
+ */
+ private int getEolCharsCount(byte [] bytes) {
+ int count = 0;
+ if (bytes != null && bytes.length >=1 &&
+ bytes[bytes.length - 1] == '\n') {
+ count++;
+ if (bytes.length >=2 && bytes[bytes.length -2] == '\r') {
+ count++;
+ }
+ }
+ return count;
+ }
+
+ public void parseHeaders() throws IOException {
+ if(parsedHeaders) {
+ return;
+ }
+
+ byte [] statusBytes = HttpParser.readRawLine(rec);
+ int eolCharCount = getEolCharsCount(statusBytes);
+ if (eolCharCount <= 0) {
+ throw new RecoverableIOException("Failed to read http status where one " +
+ " was expected: " + new String(statusBytes));
+ }
+ String statusLineStr = EncodingUtil.getString(statusBytes, 0,
+ statusBytes.length - eolCharCount, ARCConstants.DEFAULT_ENCODING);
+ if ((statusLineStr == null) ||
+ !StatusLine.startsWithHTTP(statusLineStr)) {
+ throw new RecoverableIOException("Failed parse of http status line.");
+ }
+ StatusLine statusLine = new StatusLine(statusLineStr);
+
+ this.status = statusLine.getStatusCode();
+
+ Header[] tmpHeaders = HttpParser.parseHeaders(rec,
+ ARCConstants.DEFAULT_ENCODING);
+ headers = new Hashtable<String,String>();
+ for(Header header: tmpHeaders) {
+ headers.put(header.getName(), header.getValue());
+ }
+ this.setInputStream(rec);
+ parsedHeaders = true;
+ }
+
+
+ @Override
+ public Map<String, String> getHttpHeaders() {
+ return headers;
+ }
+
+ @Override
+ public long getRecordLength() {
+ // TODO Auto-generated method stub
+ return length;
+ }
+
+ @Override
+ public int getStatusCode() {
+ return status;
+ }
+
+ @Override
+ public void close() throws IOException {
+ rec.close();
+ reader.close();
+ }
+}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
