[Archive-access-cvs] SF.net SVN: archive-access:[3523] trunk/archive-access/projects/wayback/ wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/ ZiplinedBlock.java

[<bra...@us...>]

Revision: 3523
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3523&view=rev
Author:   bradtofel
Date:     2011-09-06 04:02:57 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
FEATURE: allows multiple possible locations for a block - attempts to read will try each in order

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/ZiplinedBlock.java

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/ZiplinedBlock.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/ZiplinedBlock.java	2011-09-06 04:00:09 UTC (rev 3522)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/ZiplinedBlock.java	2011-09-06 04:02:57 UTC (rev 3523)
@@ -39,10 +39,10 @@
 			ZiplinedBlock.class.getName());
 
 	BlockLoader loader = null;
-	String urlOrPath = null;
+	String urlOrPaths[] = null;
 	long offset = -1;
-	int count = 0;
 	public final static int BLOCK_SIZE = 128 * 1024;
+	int count = BLOCK_SIZE;
 	public final static String RANGE_HEADER = "Range";
 	public final static String BYTES_HEADER = "bytes=";
 	public final static String BYTES_MINUS = "-";
@@ -50,16 +50,16 @@
 	 * @param urlOrPath URL where this file can be downloaded
 	 * @param offset start of 128K block boundary.
 	 */
-	public ZiplinedBlock(String urlOrPath, long offset) {
-		this(urlOrPath,offset,0);
+	public ZiplinedBlock(String urlOrPaths[], long offset) {
+		this(urlOrPaths,offset,BLOCK_SIZE);
 	}
 	/**
 	 * @param urlOrPath URL where this file can be downloaded
 	 * @param offset start of 128K block boundary.
 	 * @param count number of records in this block
 	 */
-	public ZiplinedBlock(String urlOrPath, long offset, int count) {
-		this.urlOrPath = urlOrPath;
+	public ZiplinedBlock(String urlOrPaths[], long offset, int count) {
+		this.urlOrPaths = urlOrPaths;
 		this.offset = offset;
 		this.count = count;
 	}
@@ -79,9 +79,24 @@
 		}
 		return readBlockInefficiently();
 	}
+	private byte[] attemptBlockLoad(BlockLoader remote) {
+		for(String urlOrPath : urlOrPaths) {
+			try {
+				return remote.getBlock(urlOrPath, offset, count);
+			} catch (IOException e) {
+				LOGGER.warning(String.format("FAILED to load(%s) (%d:%d)",
+						urlOrPath,offset,count));
+			}
+		}
+		return null;
+	}
+	
 	private BufferedReader readBlockEfficiently(BlockLoader remote)
 	throws IOException {
-		byte bytes[] = remote.getBlock(urlOrPath, offset, BLOCK_SIZE);
+		byte bytes[] = attemptBlockLoad(remote);
+		if(bytes == null) {
+			throw new IOException("Unable to load block!");
+		}
 		return new BufferedReader(new InputStreamReader(
 				new GZIPInputStream(new ByteArrayInputStream(bytes)),
 				ByteOp.UTF8));
@@ -89,10 +104,10 @@
 	private BufferedReader readBlockInefficiently() throws IOException {
 		StringBuilder sb = new StringBuilder(16);
 		sb.append(BYTES_HEADER).append(offset).append(BYTES_MINUS);
-		sb.append((offset + BLOCK_SIZE)-1);
-		LOGGER.fine("Reading block:" + urlOrPath + "("+sb.toString()+")");
+		sb.append((offset + count)-1);
+		LOGGER.fine("Reading block:" + urlOrPaths[0] + "("+sb.toString()+")");
 		// TODO: timeouts
-		URL u = new URL(urlOrPath);
+		URL u = new URL(urlOrPaths[0]);
 		URLConnection uc = u.openConnection();
 		uc.setRequestProperty(RANGE_HEADER, sb.toString());
 		return new BufferedReader(new InputStreamReader(

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.