Menu
▾
▴
archive-access-cvs
Revision: 2585
http://archive-access.svn.sourceforge.net/archive-access/?rev=2585&view=rev
Author: bradtofel
Date: 2008-08-28 21:44:41 +0000 (Thu, 28 Aug 2008)
Log Message:
-----------
COMMENT+WHITESPACE
Modified Paths:
--------------
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/SimpleResourceStore.java
Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/SimpleResourceStore.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/SimpleResourceStore.java 2008-08-25 23:30:34 UTC (rev 2584)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/SimpleResourceStore.java 2008-08-28 21:44:41 UTC (rev 2585)
@@ -35,10 +35,10 @@
/**
- * Implements ResourceStore where ARC/WARCs are accessed via HTTP 1.1 range
- * requests. All files are assumed to be "rooted" at a particular HTTP URL,
- * within a single directory, implying a file reverse-proxy to connect through
- * to actual HTTP ARC/WARC locations.
+ * Implements ResourceStore where ARC/WARCs are accessed via a local file or an
+ * HTTP 1.1 range request. All files are assumed to be "rooted" at a particular
+ * HTTP URL, or within a single local directory. The HTTP version may imply a
+ * file reverse-proxy to connect through to actual HTTP ARC/WARC locations.
*
* @author brad
* @version $Date$, $Revision$
@@ -47,7 +47,6 @@
private String prefix = null;
-
public Resource retrieveResource(CaptureSearchResult result)
throws ResourceNotAvailableException {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
Revision: 2995
http://archive-access.svn.sourceforge.net/archive-access/?rev=2995&view=rev
Author: bradtofel
Date: 2010-03-20 01:21:00 +0000 (Sat, 20 Mar 2010)
Log Message:
-----------
LOGGING: added logging when unable to access a ResourceFile
Modified Paths:
--------------
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/SimpleResourceStore.java
Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/SimpleResourceStore.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/SimpleResourceStore.java 2010-03-20 01:19:20 UTC (rev 2994)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/SimpleResourceStore.java 2010-03-20 01:21:00 UTC (rev 2995)
@@ -25,6 +25,7 @@
package org.archive.wayback.resourcestore;
import java.io.IOException;
+import java.util.logging.Logger;
import org.archive.wayback.ResourceStore;
import org.archive.wayback.core.Resource;
@@ -45,6 +46,8 @@
*/
public class SimpleResourceStore implements ResourceStore {
+ private final static Logger LOGGER = Logger.getLogger(
+ SimpleResourceStore.class.getName());
private String prefix = null;
public Resource retrieveResource(CaptureSearchResult result)
@@ -71,7 +74,7 @@
r = ResourceFactory.getResource(fileUrl, offset);
} catch (IOException e) {
-
+ LOGGER.warning("Unable to retrieve:" + fileUrl + ":" + offset);
e.printStackTrace();
throw new ResourceNotAvailableException("Unable to retrieve",
e.getLocalizedMessage());
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
Revision: 3288
http://archive-access.svn.sourceforge.net/archive-access/?rev=3288&view=rev
Author: bradtofel
Date: 2010-10-21 00:37:52 +0000 (Thu, 21 Oct 2010)
Log Message:
-----------
FEATURE: added 'retries' property which determines the number of attempts to grab a document when receiving HTTP 502 errors.
Modified Paths:
--------------
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/SimpleResourceStore.java
Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/SimpleResourceStore.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/SimpleResourceStore.java 2010-10-18 22:24:46 UTC (rev 3287)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/SimpleResourceStore.java 2010-10-21 00:37:52 UTC (rev 3288)
@@ -43,7 +43,10 @@
private final static Logger LOGGER = Logger.getLogger(
SimpleResourceStore.class.getName());
+ private static String HTTP_ERROR = "HTTP";
+ private static String HTTP_502 = "502";
private String prefix = null;
+ private int retries = 2;
public Resource retrieveResource(CaptureSearchResult result)
throws ResourceNotAvailableException {
@@ -65,9 +68,26 @@
String fileUrl = prefix + fileName;
Resource r = null;
try {
+ int attempts = retries;
+ while(attempts-- > 0) {
+ try {
+ r = ResourceFactory.getResource(fileUrl, offset);
+ break;
+ } catch (IOException e) {
+ String message = e.getMessage();
+ if(attempts > 0
+ && message.contains(HTTP_ERROR)
+ && message.contains(HTTP_502)) {
+
+ LOGGER.warning(String.format(
+ "Failed attempt for (%s) retrying with" +
+ " (%d) attempts left",fileUrl,attempts));
+ } else {
+ throw e;
+ }
+ }
+ }
- r = ResourceFactory.getResource(fileUrl, offset);
-
} catch (IOException e) {
LOGGER.warning("Unable to retrieve:" + fileUrl + ":" + offset);
e.printStackTrace();
@@ -94,4 +114,20 @@
public void shutdown() throws IOException {
// no-op
}
+
+ /**
+ * @return the number of attempts to fetch resources with an HTTP 502
+ * failure.
+ */
+ public int getRetries() {
+ return retries;
+ }
+
+ /**
+ * @param retries the number of attempts to fetch resources with an HTTP 502
+ * failure.
+ */
+ public void setRetries(int retries) {
+ this.retries = retries;
+ }
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
×
Want the latest updates on software, tech news, and AI?
Get latest updates about software, tech news, and AI from SourceForge directly in your inbox once a month.
Thanks for helping keep SourceForge clean.
X