archive-access-cvs Mailing List for Web Archive Access Utilities (Page 10)

SourceForge Headquarters 225 Broadway Suite 1600 San Diego, CA 92101 +1 (858) 422-6466

Wayback-1 - Build # 48 - Successful:

Check console output at https://builds.archive.org:1443/job/Wayback-1/48/ to view the results.



Wayback-1 - Build # 47 - Successful:

Check console output at https://builds.archive.org:1443/job/Wayback-1/47/ to view the results.



Wayback-1 - Build # 46 - Successful:

Check console output at https://builds.archive.org:1443/job/Wayback-1/46/ to view the results.



Revision: 3536
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3536&view=rev
Author:   bradtofel
Date:     2011-09-06 04:24:36 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
INITIAL REV: interface for factories which "decorate" a previously existing ReplayRenderer

Added Paths:
-----------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/replay/ReplayRendererDecoratorFactory.java

Added: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/replay/ReplayRendererDecoratorFactory.java
===================================================================

--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/replay/ReplayRendererDecoratorFactory.java	                        (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/replay/ReplayRendererDecoratorFactory.java	2011-09-06 04:24:36 UTC (rev 3536)
@@ -0,0 +1,26 @@
+/*
+ *  This file is part of the Wayback archival access software
+ *   (http://archive-access.sourceforge.net/projects/wayback/).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.archive.wayback.replay;
+
+import org.archive.wayback.ReplayRenderer;
+
+public interface ReplayRendererDecoratorFactory {
+	public ReplayRenderer decorate(ReplayRenderer renderer);
+}

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Revision: 3535
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3535&view=rev
Author:   bradtofel
Date:     2011-09-06 04:23:35 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
FEATURE: made no-argument constructor, allowing configuration of decorated via getter/setter

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/replay/ReplayRendererDecorator.java

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/replay/ReplayRendererDecorator.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/replay/ReplayRendererDecorator.java	2011-09-06 04:22:06 UTC (rev 3534)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/replay/ReplayRendererDecorator.java	2011-09-06 04:23:35 UTC (rev 3535)
@@ -43,7 +43,15 @@
 public abstract class ReplayRendererDecorator implements ReplayRenderer {
 
 	protected ReplayRenderer decorated = null;
+	public ReplayRendererDecorator() {
+	}
 	/**
+	 * @param httpHeaderProcessor
+	 */
+	public ReplayRendererDecorator(ReplayRenderer decorated) {
+		this.decorated = decorated;
+	}
+	/**
 	 * @return the decorated
 	 */
 	public ReplayRenderer getDecorated() {
@@ -55,14 +63,6 @@
 	public void setDecorated(ReplayRenderer decorated) {
 		this.decorated = decorated;
 	}
-	public ReplayRendererDecorator() {
-	}
-	/**
-	 * @param httpHeaderProcessor
-	 */
-	public ReplayRendererDecorator(ReplayRenderer decorated) {
-		this.decorated = decorated;
-	}
 
 	public abstract void renderResource(HttpServletRequest httpRequest,
 			HttpServletResponse httpResponse, WaybackRequest wbRequest,

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Revision: 3534
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3534&view=rev
Author:   bradtofel
Date:     2011-09-06 04:22:06 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
FEATURE: now implements AccessPointAware, to (hopefully) simplify configuration

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/archivalurl/ArchivalUrlResultURIConverter.java

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/archivalurl/ArchivalUrlResultURIConverter.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/archivalurl/ArchivalUrlResultURIConverter.java	2011-09-06 04:20:57 UTC (rev 3533)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/archivalurl/ArchivalUrlResultURIConverter.java	2011-09-06 04:22:06 UTC (rev 3534)
@@ -21,6 +21,8 @@
 
 import org.archive.wayback.ResultURIConverter;
 import org.archive.wayback.util.url.UrlOperations;
+import org.archive.wayback.webapp.AccessPoint;
+import org.archive.wayback.webapp.AccessPointAware;
 
 /**
  *
@@ -28,7 +30,8 @@
  * @author brad
  * @version $Date$, $Revision$
  */
-public class ArchivalUrlResultURIConverter implements ResultURIConverter {
+public class ArchivalUrlResultURIConverter implements ResultURIConverter, 
+	AccessPointAware {
 	/**
 	 * configuration name for URL prefix of replay server
 	 */
@@ -71,4 +74,10 @@
 	public String getReplayURIPrefix() {
 		return replayURIPrefix;
 	}
+
+	public void setAccessPoint(AccessPoint accessPoint) {
+		if(replayURIPrefix == null) {
+			replayURIPrefix = accessPoint.getReplayPrefix();
+		}
+	}
 }

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Revision: 3533
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3533&view=rev
Author:   bradtofel
Date:     2011-09-06 04:20:57 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
FEATURE: checks with AccessPoint before forcing "clean queries"

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/archivalurl/requestparser/ArchivalUrlFormRequestParser.java

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/archivalurl/requestparser/ArchivalUrlFormRequestParser.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/archivalurl/requestparser/ArchivalUrlFormRequestParser.java	2011-09-06 04:20:15 UTC (rev 3532)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/archivalurl/requestparser/ArchivalUrlFormRequestParser.java	2011-09-06 04:20:57 UTC (rev 3533)
@@ -54,12 +54,14 @@
 				accessPoint.translateRequestPathQuery(httpRequest);
 			ArchivalUrl aUrl = new ArchivalUrl(wbRequest);
 			String bestPath = aUrl.toString();
-			if(!bestPath.equals(requestPath)) {
-				String betterURI = (wbRequest.isReplayRequest() ? 
-						accessPoint.getReplayPrefix() : 
-							accessPoint.getQueryPrefix()) 
-						+ bestPath;
-				throw new BetterRequestException(betterURI);
+			if(accessPoint.isForceCleanQueries()) {
+				if(!bestPath.equals(requestPath)) {
+					String betterURI = (wbRequest.isReplayRequest() ? 
+							accessPoint.getReplayPrefix() : 
+								accessPoint.getQueryPrefix()) 
+							+ bestPath;
+					throw new BetterRequestException(betterURI);
+				}
 			}
 		}
 		return wbRequest;

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Revision: 3532
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3532&view=rev
Author:   bradtofel
Date:     2011-09-06 04:20:15 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
INITIAL REV: class which attempts to efficiently track the first, prev, closest, next, and last results in a set, relative to a specific date

Added Paths:
-----------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/partition/NotableResultExtractor.java

Added: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/partition/NotableResultExtractor.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/partition/NotableResultExtractor.java	                        (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/partition/NotableResultExtractor.java	2011-09-06 04:20:15 UTC (rev 3532)
@@ -0,0 +1,153 @@
+/*
+ *  This file is part of the Wayback archival access software
+ *   (http://archive-access.sourceforge.net/projects/wayback/).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.archive.wayback.partition;
+
+import java.util.Date;
+
+import org.archive.wayback.core.CaptureSearchResult;
+import org.archive.wayback.util.ObjectFilter;
+
+/**
+ * ObjectFilter of CaptureSearchResult objects, which includes all results
+ * but keeps track of "notable" results, in particular, those relative to a
+ * specific date: first, previous, closest, next, and last
+ * @author brad
+ *
+ */
+public class NotableResultExtractor implements ObjectFilter<CaptureSearchResult>{
+	private Date want;
+	
+	private long closestDist = -1;
+	private CaptureSearchResult first;
+	private CaptureSearchResult last;
+	private CaptureSearchResult prev;
+	private CaptureSearchResult closest;
+	private CaptureSearchResult next;
+	
+	public NotableResultExtractor(Date want) {
+		this.want = want;
+	}
+	public int filterObject(CaptureSearchResult o) {
+		Date oDate = o.getCaptureDate();
+		long tmp = oDate.getTime();
+		long cmp = oDate.getTime() - want.getTime();
+		long abs = Math.abs(cmp);
+		
+		last = o;
+		if(first == null) {
+			// special case for the first record:
+			first = o;
+			closest = o;
+			closestDist = abs;
+
+		} else if(next != null) {
+			// we already found everything interesting - no op:
+		} else {
+
+			// is this after "want"?
+			if(cmp > 0) {
+				// is it closer than our last closest?
+				if(abs < closestDist) {
+					// after, but closer:
+					// if closest was before want, it's now prev:
+					if(closest.getCaptureDate().before(want)) {
+						prev = closest;
+					}
+					closest = o;
+					closestDist = abs;
+				} else {
+					// we've already found the closest, this is next:
+					next = o;
+				}
+			} else {
+				// before "want" & now the closest, old closest is now prev
+				prev = closest;
+				closest = o;
+				closestDist = abs;
+			}
+		}
+		return FILTER_INCLUDE;
+	}
+	public void complete() {
+		
+	}
+	
+	/**
+	 * @return the prev
+	 */
+	public CaptureSearchResult getPrev() {
+		return prev;
+	}
+	/**
+	 * @param prev the prev to set
+	 */
+	public void setPrev(CaptureSearchResult prev) {
+		this.prev = prev;
+	}
+	/**
+	 * @return the closest
+	 */
+	public CaptureSearchResult getClosest() {
+		return closest;
+	}
+	/**
+	 * @param closest the closest to set
+	 */
+	public void setClosest(CaptureSearchResult closest) {
+		this.closest = closest;
+	}
+	/**
+	 * @return the next
+	 */
+	public CaptureSearchResult getNext() {
+		return next;
+	}
+	/**
+	 * @param next the next to set
+	 */
+	public void setNext(CaptureSearchResult next) {
+		this.next = next;
+	}
+	/**
+	 * @return the first
+	 */
+	public CaptureSearchResult getFirst() {
+		return first;
+	}
+	/**
+	 * @param first the first to set
+	 */
+	public void setFirst(CaptureSearchResult first) {
+		this.first = first;
+	}
+	/**
+	 * @return the last
+	 */
+	public CaptureSearchResult getLast() {
+		return last;
+	}
+	/**
+	 * @param last the last to set
+	 */
+	public void setLast(CaptureSearchResult last) {
+		this.last = last;
+	}
+
+}

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Wayback-1 - Build # 45 - Fixed:

Check console output at https://builds.archive.org:1443/job/Wayback-1/45/ to view the results.



Revision: 3531
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3531&view=rev
Author:   bradtofel
Date:     2011-09-06 04:18:00 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
INITIAL REV: BlockLoader which loads from Local RandomAccessFiles, and a generic BlockLoader which tries to do "the right thing" with each location, choosing the correct block loader based on the URL/String prefix

Added Paths:
-----------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/GenericBlockLoader.java
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/LocalFileBlockLoader.java

Added: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/GenericBlockLoader.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/GenericBlockLoader.java	                        (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/GenericBlockLoader.java	2011-09-06 04:18:00 UTC (rev 3531)
@@ -0,0 +1,78 @@
+/*
+ *  This file is part of the Wayback archival access software
+ *   (http://archive-access.sourceforge.net/projects/wayback/).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.archive.wayback.resourceindex.ziplines;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+/**
+ * Generic BlockLoader, which may simplify configuration - inspecting each 
+ * location to attempt to choose the correct BlockLoader:
+ *   HDFS, HTTP, or LocalFile
+ * @author brad
+ *
+ */
+public class GenericBlockLoader implements BlockLoader {
+	Http11BlockLoader http = null;
+	HDFSBlockLoader hdfs = null;
+	LocalFileBlockLoader local = null;
+	private String defaultFSURI;
+	public GenericBlockLoader() {
+		http = new Http11BlockLoader();
+//		hdfs = new HDFSBlockLoader(null);
+//		try {
+//			hdfs.init();
+//		} catch (IOException e) {
+//			// TODO Auto-generated catch block
+//			e.printStackTrace();
+//		} catch (URISyntaxException e) {
+//			// TODO Auto-generated catch block
+//			e.printStackTrace();
+//		}
+		local = new LocalFileBlockLoader();
+	}
+	public void init() {
+		if(defaultFSURI != null) {
+			hdfs = new HDFSBlockLoader(defaultFSURI);
+			try {
+				hdfs.init();
+			} catch (IOException e) {
+				// TODO Auto-generated catch block
+				e.printStackTrace();
+			} catch (URISyntaxException e) {
+				// TODO Auto-generated catch block
+				e.printStackTrace();
+			}
+		}
+	}
+	public byte[] getBlock(String url, long offset, int length)
+			throws IOException {
+		if(hdfs != null && url.startsWith("hdfs://")) {
+			return hdfs.getBlock(url, offset, length);
+		} else if(url.startsWith("/")) {
+			return local.getBlock(url, offset, length);
+		}
+		return http.getBlock(url, offset, length);
+	}
+	public void setDefaultFSURI(String uri) {
+		defaultFSURI = uri;
+//		hdfs.setDefaultFSURI(uri);
+	}
+}

Added: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/LocalFileBlockLoader.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/LocalFileBlockLoader.java	                        (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/LocalFileBlockLoader.java	2011-09-06 04:18:00 UTC (rev 3531)
@@ -0,0 +1,47 @@
+/*
+ *  This file is part of the Wayback archival access software
+ *   (http://archive-access.sourceforge.net/projects/wayback/).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.archive.wayback.resourceindex.ziplines;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+
+/**
+ * Simple block loader which uses RandomAccessFiles to grab ranges of local 
+ * files.
+ * @author brad
+ *
+ */
+public class LocalFileBlockLoader implements BlockLoader {
+
+	public byte[] getBlock(String url, long offset, int length)
+			throws IOException {
+		File file = new File(url);
+		RandomAccessFile raf = new RandomAccessFile(file, "r");
+		raf.seek(offset);
+		if(raf.getFilePointer() != offset) {
+			throw new IOException("Failed seek("+offset+") in ("+url+")");
+		}
+		byte b[] = new byte[length];
+		raf.readFully(b);
+		return b;
+	}
+
+}

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Revision: 3530
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3530&view=rev
Author:   bradtofel
Date:     2011-09-06 04:15:02 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
FEATURE:
  - now allows -flexFormat in command line usage
  - allows configuration of multiple locations of each block source file
  - improved logging of errors

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/ZiplinesSearchResultSource.java

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/ZiplinesSearchResultSource.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/ZiplinesSearchResultSource.java	2011-09-06 04:12:58 UTC (rev 3529)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/ZiplinesSearchResultSource.java	2011-09-06 04:15:02 UTC (rev 3530)
@@ -32,6 +32,7 @@
 import org.archive.wayback.exception.ResourceIndexNotAvailableException;
 import org.archive.wayback.resourceindex.SearchResultSource;
 import org.archive.wayback.resourceindex.cdx.CDXFormatToSearchResultAdapter;
+import org.archive.wayback.resourceindex.cdx.format.CDXFlexFormat;
 import org.archive.wayback.resourceindex.cdx.format.CDXFormat;
 import org.archive.wayback.resourceindex.cdx.format.CDXFormatException;
 import org.archive.wayback.util.AdaptedIterator;
@@ -76,7 +77,7 @@
 	 * Local path containing URL for each CHUNK
 	 */
 	private String chunkMapPath = null;
-	private HashMap<String,String> chunkMap = null;
+	private HashMap<String,BlockLocation> chunkMap = null;
 	private CDXFormat format = null;
 	private int maxBlocks = 1000;
 	private BlockLoader blockLoader = null;
@@ -87,19 +88,25 @@
 		this.format = format;
 	}
 	public void init() throws IOException {
-		chunkMap = new HashMap<String, String>();
+		chunkMap = new HashMap<String, BlockLocation>();
 		FlatFile ff = new FlatFile(chunkMapPath);
 		CloseableIterator<String> lines = ff.getSequentialIterator();
 		while(lines.hasNext()) {
 			String line = lines.next();
 			String[] parts = line.split("\\s");
-			if(parts.length != 2) {
+			if(parts.length < 2) {
 				LOGGER.severe("Bad line(" + line +") in (" + 
 						chunkMapPath + ")");
 				throw new IOException("Bad line(" + line +") in (" + 
 						chunkMapPath + ")");
 			}
-			chunkMap.put(parts[0],parts[1]);
+			
+			String locations[] = new String[parts.length - 1];
+			for(int i = 1; i < parts.length; i++) {
+				locations[i-1] = parts[i];
+			}
+			BlockLocation bl = new BlockLocation(parts[0], locations);
+			chunkMap.put(parts[0],bl);
 		}
 		lines.close();
 		chunkIndex = new FlatFile(chunkIndexPath);
@@ -141,13 +148,14 @@
 			itr = chunkIndex.getRecordIteratorLT(prefix);
 			while(itr.hasNext()) {
 				if(numBlocks >= maxBlocks) {
+					LOGGER.warning("Truncated by blocks for " + prefix);
 					truncated = true;
 					break;
 				}
 				String blockDescriptor = itr.next();
 				numBlocks++;
 				String parts[] = blockDescriptor.split("\t");
-				if(parts.length != 3) {
+				if((parts.length < 3) || (parts.length > 4)) {
 					LOGGER.severe("Bad line(" + blockDescriptor +") in (" + 
 							chunkMapPath + ")");
 					throw new ResourceIndexNotAvailableException("Bad line(" + 
@@ -164,10 +172,22 @@
 					break;
 				}
 				// add this and keep lookin...
-				String url = chunkMap.get(parts[1]);
+				BlockLocation bl = chunkMap.get(parts[1]);
+				if(bl == null) {
+					LOGGER.severe("No locations for block(" + parts[1] +")");
+					throw new ResourceIndexNotAvailableException(
+							"No locations for block(" + parts[1] + ")");
+				}
 				long offset = Long.parseLong(parts[2]);
-				LOGGER.info("Adding block source(" + parts[1] + "):" + offset);
-				ZiplinedBlock block = new ZiplinedBlock(url, offset);
+				ZiplinedBlock block;
+				if(parts.length == 3) {
+					LOGGER.info("Adding block source(" + parts[1] + "):" + offset);
+					block = new ZiplinedBlock(bl.getLocations(), offset);
+				} else {
+					int length = Integer.parseInt(parts[3]);
+					LOGGER.info("Adding block source(" + parts[1] + "):" + offset + " - " + length);
+					block = new ZiplinedBlock(bl.getLocations(), offset, length);
+				}
 				block.setLoader(blockLoader);
 				blocks.add(block);
 			}
@@ -308,6 +328,13 @@
 					e1.printStackTrace();
 					System.exit(1);
 				}
+			} else if(args[idx].equals("-flexFormat")) {
+				try {
+					zl.setFormat(new CDXFlexFormat(" CDX A"));
+				} catch (CDXFormatException e1) {
+					e1.printStackTrace();
+					System.exit(1);
+				}
 			} else if(args[idx].equals("-blockDump")) {
 				blockDump = true;
 			} else if(args[idx].equals("-hdfs")) {
@@ -366,7 +393,7 @@
 				
 				ArrayList<ZiplinedBlock> blocks = zl.getBlockListForPrefix(key);
 				for(ZiplinedBlock block : blocks) {
-					pw.format("%s\t%s\n", block.urlOrPath, block.offset);
+					pw.format("%s\t%s\n", block.urlOrPaths[0], block.offset);
 				}
 				pw.close();
 

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Revision: 3529
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3529&view=rev
Author:   bradtofel
Date:     2011-09-06 04:12:58 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
INITIAL REV: ObjectFilter which excludes filters that both match a certain prefix AND are newer than a specific period

Added Paths:
-----------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/filters/FilePrefixDateEmbargoFilter.java

Added: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/filters/FilePrefixDateEmbargoFilter.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/filters/FilePrefixDateEmbargoFilter.java	                        (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/filters/FilePrefixDateEmbargoFilter.java	2011-09-06 04:12:58 UTC (rev 3529)
@@ -0,0 +1,49 @@
+/*
+ *  This file is part of the Wayback archival access software
+ *   (http://archive-access.sourceforge.net/projects/wayback/).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.archive.wayback.resourceindex.filters;
+
+import java.util.Date;
+
+import org.archive.wayback.core.CaptureSearchResult;
+import org.archive.wayback.util.ObjectFilter;
+
+/**
+ * Blocks only files matching a given prefix, iff they are newer than a given
+ * embargo period.
+ * 
+ * @author brad
+ *
+ */
+public class FilePrefixDateEmbargoFilter  implements ObjectFilter<CaptureSearchResult> {
+	protected String matchPrefix = null;
+	protected Date embargoDate = null;
+	public FilePrefixDateEmbargoFilter(String prefix, long minAge) {
+		matchPrefix = prefix;
+		embargoDate = new Date(System.currentTimeMillis() - minAge);
+	}
+	public int filterObject(CaptureSearchResult o) {
+		if(!o.getFile().startsWith(matchPrefix)) {
+			return FILTER_INCLUDE;
+		}
+		return o.getCaptureDate().compareTo(embargoDate) < 0 
+			? FILTER_INCLUDE : FILTER_EXCLUDE;
+	}
+	
+}

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Revision: 3528
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3528&view=rev
Author:   bradtofel
Date:     2011-09-06 04:11:53 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
INITIAL REV: simple composite filter

Added Paths:
-----------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/filters/CompositeFilter.java

Added: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/filters/CompositeFilter.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/filters/CompositeFilter.java	                        (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/filters/CompositeFilter.java	2011-09-06 04:11:53 UTC (rev 3528)
@@ -0,0 +1,64 @@
+/*
+ *  This file is part of the Wayback archival access software
+ *   (http://archive-access.sourceforge.net/projects/wayback/).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.archive.wayback.resourceindex.filters;
+
+import java.util.List;
+
+import org.archive.wayback.core.CaptureSearchResult;
+import org.archive.wayback.util.ObjectFilter;
+
+/**
+ * Simple composite ObjectFilter - which includes only if all components include
+ * 
+ * @author brad
+ *
+ */
+public class CompositeFilter implements ObjectFilter<CaptureSearchResult> {
+	private List<ObjectFilter<CaptureSearchResult>> filters;
+	/* (non-Javadoc)
+	 * @see org.archive.wayback.resourceindex.SearchResultFilter#filterSearchResult(org.archive.wayback.core.SearchResult)
+	 */
+	public int filterObject(CaptureSearchResult r) {
+		
+		for(ObjectFilter<CaptureSearchResult> filter : filters) {
+			if(filter == null) {
+				return FILTER_EXCLUDE;
+			}
+			int result = filter.filterObject(r);
+			if(result != FILTER_INCLUDE) {
+				return result;
+			}
+		}
+		return FILTER_INCLUDE;
+	}
+	/**
+	 * @return the filters
+	 */
+	public List<ObjectFilter<CaptureSearchResult>> getFilters() {
+		return filters;
+	}
+	/**
+	 * @param filters the filters to set
+	 */
+	public void setFilters(List<ObjectFilter<CaptureSearchResult>> filters) {
+		this.filters = filters;
+	}
+
+}

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Wayback-1 - Build # 44 - Still Failing:

Check console output at https://builds.archive.org:1443/job/Wayback-1/44/ to view the results.



Revision: 3527
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3527&view=rev
Author:   bradtofel
Date:     2011-09-06 04:09:48 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
FEATURE: checks AccessPoint for a possible CustomResultFilterFactory - and adds the resulting filter from that factory to the filter list, if present

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/filterfactory/AccessPointCaptureFilterGroup.java

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/filterfactory/AccessPointCaptureFilterGroup.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/filterfactory/AccessPointCaptureFilterGroup.java	2011-09-06 04:08:28 UTC (rev 3526)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/filterfactory/AccessPointCaptureFilterGroup.java	2011-09-06 04:09:48 UTC (rev 3527)
@@ -33,6 +33,7 @@
 import org.archive.wayback.util.ObjectFilter;
 import org.archive.wayback.util.ObjectFilterChain;
 import org.archive.wayback.webapp.AccessPoint;
+import org.archive.wayback.webapp.CustomResultFilterFactory;
 
 public class AccessPointCaptureFilterGroup implements CaptureFilterGroup {
 	private ObjectFilterChain<CaptureSearchResult> chain = null;
@@ -68,6 +69,14 @@
 			if(embargoMS > 0) {
 				chain.addFilter(new DateEmbargoFilter(embargoMS));
 			}
+			CustomResultFilterFactory factory = accessPoint.getFilterFactory();
+			if(factory != null) {
+				ObjectFilter<CaptureSearchResult> filter = 
+					factory.get(accessPoint);
+				if(filter != null) {
+					chain.addFilter(filter);
+				}
+			}
 		}
 	}
 	

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Revision: 3526
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3526&view=rev
Author:   bradtofel
Date:     2011-09-06 04:08:28 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
HACK: subclass of CDXFormat - which simply does old style CDX line parsing - allowing either 9 or 10 columns. This allows a CDXFormatIndex to be used with a CDX file containing both 9 and 10 columns in various lines

Added Paths:
-----------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/cdx/format/CDXFlexFormat.java

Added: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/cdx/format/CDXFlexFormat.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/cdx/format/CDXFlexFormat.java	                        (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/cdx/format/CDXFlexFormat.java	2011-09-06 04:08:28 UTC (rev 3526)
@@ -0,0 +1,100 @@
+package org.archive.wayback.resourceindex.cdx.format;
+
+import java.util.logging.Logger;
+
+import org.archive.wayback.core.CaptureSearchResult;
+import org.archive.wayback.util.url.UrlOperations;
+
+public class CDXFlexFormat extends CDXFormat {
+	private final static String SCHEME_STRING = "://";
+	private final static String DEFAULT_SCHEME = "http://";
+
+	private static final Logger LOGGER = 
+		Logger.getLogger(CDXFlexFormat.class.getName());
+	public CDXFlexFormat(String cdxSpec) throws CDXFormatException {
+		super(cdxSpec);
+	}
+	private static int getEndOfHostIndex(String url) {
+		int portIdx = url.indexOf(UrlOperations.PORT_SEPARATOR);
+		int pathIdx = url.indexOf(UrlOperations.PATH_START);
+		if(portIdx == -1 && pathIdx == -1) {
+			return url.length();
+		}
+		if(portIdx == -1) {
+			return pathIdx;
+		}
+		if(pathIdx == -1) {
+			return portIdx;
+		}
+		if(pathIdx > portIdx) {
+			return portIdx;
+		} else {
+			return pathIdx;
+		}
+	}
+
+	/* (non-Javadoc)
+	 * @see org.archive.wayback.resourceindex.cdx.format.CDXFormat#parseResult(java.lang.String)
+	 */
+	@Override
+	public CaptureSearchResult parseResult(String line)
+			throws CDXFormatException {
+		CaptureSearchResult result = new CaptureSearchResult();
+		String[] tokens = line.split(" ");
+		boolean hasRobotFlags = false;
+		if (tokens.length != 9) {
+			if(tokens.length == 10) {
+				hasRobotFlags = true;
+			} else {
+				return null;
+			}
+			//throw new IllegalArgumentException("Need 9 columns("+line+")");
+		}
+		String urlKey = tokens[0];
+		String captureTS = tokens[1];
+		String originalUrl = tokens[2];
+		
+		// convert from ORIG_HOST to ORIG_URL here:
+		if(!originalUrl.contains(SCHEME_STRING)) {
+			StringBuilder sb = new StringBuilder(urlKey.length());
+			sb.append(DEFAULT_SCHEME);
+			sb.append(originalUrl);
+			sb.append(urlKey.substring(getEndOfHostIndex(urlKey)));
+			originalUrl = sb.toString();
+		}
+		String mimeType = tokens[3];
+		String httpCode = tokens[4];
+		String digest = tokens[5];
+		String redirectUrl = tokens[6];
+		long compressedOffset = -1;
+		int nextToken = 7;
+		if(hasRobotFlags) {
+			result.setRobotFlags(tokens[nextToken]);
+			nextToken++;
+		}
+
+		if(!tokens[nextToken].equals("-")) {
+			try {
+				compressedOffset = Long.parseLong(tokens[nextToken]);
+			} catch (NumberFormatException e) {
+				LOGGER.warning("Bad compressed Offset field("+nextToken+") in (" +
+						line +")");
+				return null;
+			}
+		}
+		nextToken++;
+		String fileName = tokens[nextToken];
+		result.setUrlKey(urlKey);
+		result.setCaptureTimestamp(captureTS);
+		result.setOriginalUrl(originalUrl);
+		result.setMimeType(mimeType);
+		result.setHttpCode(httpCode);
+		result.setDigest(digest);
+		result.setRedirectUrl(redirectUrl);
+		result.setOffset(compressedOffset);
+		result.setFile(fileName);
+
+		return result;
+	}
+
+}

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Revision: 3525
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3525&view=rev
Author:   bradtofel
Date:     2011-09-06 04:07:08 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
SCOPE - made getField() protected

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/cdx/format/CDXFormat.java

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/cdx/format/CDXFormat.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/cdx/format/CDXFormat.java	2011-09-06 04:06:09 UTC (rev 3524)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/cdx/format/CDXFormat.java	2011-09-06 04:07:08 UTC (rev 3525)
@@ -133,7 +133,7 @@
 		delimiterS = new String(""+delimiter);
 	}
 	
-	private CDXField getField(char fieldChar) throws CDXFormatException {
+	protected CDXField getField(char fieldChar) throws CDXFormatException {
 		CDXField field = null;
 		switch (fieldChar) {
 			case 'A':  field = new URLKeyCDXField(); break;

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Wayback-1 - Build # 43 - Still Failing:

Check console output at https://builds.archive.org:1443/job/Wayback-1/43/ to view the results.



Revision: 3524
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3524&view=rev
Author:   bradtofel
Date:     2011-09-06 04:06:09 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
FEATURE: allows multiple possible locations for a block - attempts to read will try each in order

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/ZiplinedBlockStringSequence.java

Added Paths:
-----------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/BlockLocation.java

Added: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/BlockLocation.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/BlockLocation.java	                        (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/BlockLocation.java	2011-09-06 04:06:09 UTC (rev 3524)
@@ -0,0 +1,41 @@
+/*
+ *  This file is part of the Wayback archival access software
+ *   (http://archive-access.sourceforge.net/projects/wayback/).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.archive.wayback.resourceindex.ziplines;
+
+public class BlockLocation {
+	String name;
+	String locations[];
+	public BlockLocation(String name, String locations[]) {
+		this.name = name;
+		this.locations = locations;
+	}
+	/**
+	 * @return the name
+	 */
+	public String getName() {
+		return name;
+	}
+	/**
+	 * @return the locations
+	 */
+	public String[] getLocations() {
+		return locations;
+	}
+}

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/ZiplinedBlockStringSequence.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/ZiplinedBlockStringSequence.java	2011-09-06 04:02:57 UTC (rev 3523)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/ZiplinedBlockStringSequence.java	2011-09-06 04:06:09 UTC (rev 3524)
@@ -37,11 +37,11 @@
 			ZiplinedBlockStringSequence.class.getName());
 
 	private FlatFile chunkIndex = null;
-	private HashMap<String,String> chunkMap = null;
+	private HashMap<String,BlockLocation> chunkMap = null;
 	private int maxBlocks = 10000;
 
 	public ZiplinedBlockStringSequence(FlatFile chunkIndex, 
-			HashMap<String,String> chunkMap) {
+			HashMap<String,BlockLocation> chunkMap) {
 		this.chunkIndex = chunkIndex;
 		this.chunkMap = chunkMap;
 	}
@@ -80,11 +80,11 @@
 					break;
 				}
 				// add this and keep lookin...
-				String url = chunkMap.get(parts[1]);
+				BlockLocation bl = chunkMap.get(parts[1]);
 				long offset = Long.parseLong(parts[2]);
 				int count = Integer.parseInt(parts[3]);
 				
-				blocks.add(new ZiplinedBlock(url, offset, count));
+				blocks.add(new ZiplinedBlock(bl.getLocations(), offset, count));
 			}
 		} finally {
 			if(itr != null) {

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Revision: 3523
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3523&view=rev
Author:   bradtofel
Date:     2011-09-06 04:02:57 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
FEATURE: allows multiple possible locations for a block - attempts to read will try each in order

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/ZiplinedBlock.java

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/ZiplinedBlock.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/ZiplinedBlock.java	2011-09-06 04:00:09 UTC (rev 3522)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/ZiplinedBlock.java	2011-09-06 04:02:57 UTC (rev 3523)
@@ -39,10 +39,10 @@
 			ZiplinedBlock.class.getName());
 
 	BlockLoader loader = null;
-	String urlOrPath = null;
+	String urlOrPaths[] = null;
 	long offset = -1;
-	int count = 0;
 	public final static int BLOCK_SIZE = 128 * 1024;
+	int count = BLOCK_SIZE;
 	public final static String RANGE_HEADER = "Range";
 	public final static String BYTES_HEADER = "bytes=";
 	public final static String BYTES_MINUS = "-";
@@ -50,16 +50,16 @@
 	 * @param urlOrPath URL where this file can be downloaded
 	 * @param offset start of 128K block boundary.
 	 */
-	public ZiplinedBlock(String urlOrPath, long offset) {
-		this(urlOrPath,offset,0);
+	public ZiplinedBlock(String urlOrPaths[], long offset) {
+		this(urlOrPaths,offset,BLOCK_SIZE);
 	}
 	/**
 	 * @param urlOrPath URL where this file can be downloaded
 	 * @param offset start of 128K block boundary.
 	 * @param count number of records in this block
 	 */
-	public ZiplinedBlock(String urlOrPath, long offset, int count) {
-		this.urlOrPath = urlOrPath;
+	public ZiplinedBlock(String urlOrPaths[], long offset, int count) {
+		this.urlOrPaths = urlOrPaths;
 		this.offset = offset;
 		this.count = count;
 	}
@@ -79,9 +79,24 @@
 		}
 		return readBlockInefficiently();
 	}
+	private byte[] attemptBlockLoad(BlockLoader remote) {
+		for(String urlOrPath : urlOrPaths) {
+			try {
+				return remote.getBlock(urlOrPath, offset, count);
+			} catch (IOException e) {
+				LOGGER.warning(String.format("FAILED to load(%s) (%d:%d)",
+						urlOrPath,offset,count));
+			}
+		}
+		return null;
+	}
+	
 	private BufferedReader readBlockEfficiently(BlockLoader remote)
 	throws IOException {
-		byte bytes[] = remote.getBlock(urlOrPath, offset, BLOCK_SIZE);
+		byte bytes[] = attemptBlockLoad(remote);
+		if(bytes == null) {
+			throw new IOException("Unable to load block!");
+		}
 		return new BufferedReader(new InputStreamReader(
 				new GZIPInputStream(new ByteArrayInputStream(bytes)),
 				ByteOp.UTF8));
@@ -89,10 +104,10 @@
 	private BufferedReader readBlockInefficiently() throws IOException {
 		StringBuilder sb = new StringBuilder(16);
 		sb.append(BYTES_HEADER).append(offset).append(BYTES_MINUS);
-		sb.append((offset + BLOCK_SIZE)-1);
-		LOGGER.fine("Reading block:" + urlOrPath + "("+sb.toString()+")");
+		sb.append((offset + count)-1);
+		LOGGER.fine("Reading block:" + urlOrPaths[0] + "("+sb.toString()+")");
 		// TODO: timeouts
-		URL u = new URL(urlOrPath);
+		URL u = new URL(urlOrPaths[0]);
 		URLConnection uc = u.openConnection();
 		uc.setRequestProperty(RANGE_HEADER, sb.toString());
 		return new BufferedReader(new InputStreamReader(

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Wayback-1 - Build # 42 - Still Failing:

Check console output at https://builds.archive.org:1443/job/Wayback-1/42/ to view the results.



Revision: 3522
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3522&view=rev
Author:   bradtofel
Date:     2011-09-06 04:00:09 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
LICENSE

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/BlockLoader.java

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/BlockLoader.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/BlockLoader.java	2011-09-06 03:59:24 UTC (rev 3521)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/BlockLoader.java	2011-09-06 04:00:09 UTC (rev 3522)
@@ -1,3 +1,22 @@
+/*
+ *  This file is part of the Wayback archival access software
+ *   (http://archive-access.sourceforge.net/projects/wayback/).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
 package org.archive.wayback.resourceindex.ziplines;
 
 import java.io.IOException;

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Revision: 3521
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3521&view=rev
Author:   bradtofel
Date:     2011-09-06 03:59:24 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
LICENSE

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/HDFSBlockLoader.java
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/Http11BlockLoader.java

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/HDFSBlockLoader.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/HDFSBlockLoader.java	2011-09-06 03:58:31 UTC (rev 3520)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/HDFSBlockLoader.java	2011-09-06 03:59:24 UTC (rev 3521)
@@ -1,3 +1,22 @@
+/*
+ *  This file is part of the Wayback archival access software
+ *   (http://archive-access.sourceforge.net/projects/wayback/).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
 package org.archive.wayback.resourceindex.ziplines;
 
 import java.io.IOException;

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/Http11BlockLoader.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/Http11BlockLoader.java	2011-09-06 03:58:31 UTC (rev 3520)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/Http11BlockLoader.java	2011-09-06 03:59:24 UTC (rev 3521)
@@ -1,3 +1,22 @@
+/*
+ *  This file is part of the Wayback archival access software
+ *   (http://archive-access.sourceforge.net/projects/wayback/).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
 package org.archive.wayback.resourceindex.ziplines;
 
 import java.io.IOException;

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Revision: 3520
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3520&view=rev
Author:   bradtofel
Date:     2011-09-06 03:58:31 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
LOGGING

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/Http11BlockLoader.java

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/Http11BlockLoader.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/Http11BlockLoader.java	2011-09-06 03:57:31 UTC (rev 3519)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/Http11BlockLoader.java	2011-09-06 03:58:31 UTC (rev 3520)
@@ -61,7 +61,7 @@
 		try {
 			method = new GetMethod(url);
 		} catch(IllegalArgumentException e) {
-			LOGGER.warning("Bad URL for live web fetch:" + url);
+			LOGGER.warning("Bad URL for block fetch:" + url);
 			throw new IOException("Url:" + url + " does not look like an URL?");
 		}
 		StringBuilder sb = new StringBuilder(16);

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Revision: 3519
          http://archive-access.svn.sourceforge.net/archive-access/?rev=3519&view=rev
Author:   bradtofel
Date:     2011-09-06 03:57:31 +0000 (Tue, 06 Sep 2011)
Log Message:
-----------
FEATURE: a sad attempt to allow construction without a given HDFS URL

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/HDFSBlockLoader.java

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/HDFSBlockLoader.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/HDFSBlockLoader.java	2011-09-06 03:52:41 UTC (rev 3518)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/ziplines/HDFSBlockLoader.java	2011-09-06 03:57:31 UTC (rev 3519)
@@ -16,9 +16,17 @@
 		this.defaultFSURI = defaultFSURI;
 	}
 	public void init() throws IOException, URISyntaxException {
-		Configuration c = new Configuration();
-		c.set("fs.default.name",defaultFSURI);
-		fs = FileSystem.get(new URI(defaultFSURI),c);
+		if(defaultFSURI == null) {
+			Configuration c = new Configuration();
+//			String defaultURI = "hdfs://hadoop-name.example.org/";
+//			c.set("fs.default.name",defaultURI);
+//			fs = FileSystem.get(new URI(defaultURI),c);		
+			fs = FileSystem.get(c);
+		} else {
+			Configuration c = new Configuration();
+			c.set("fs.default.name",defaultFSURI);
+			fs = FileSystem.get(new URI(defaultFSURI),c);
+		}
 	}
 
 	public byte[] getBlock(String url, long offset, int length)
@@ -42,5 +50,16 @@
 	 */
 	public void setDefaultFSURI(String defaultFSURI) {
 		this.defaultFSURI = defaultFSURI;
+		Configuration c = new Configuration();
+		c.set("fs.default.name",defaultFSURI);
+		try {
+			fs = FileSystem.get(new URI(defaultFSURI),c);
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (URISyntaxException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
 	}
 }

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.





2005	Jan	Feb	Mar	Apr	May	Jun	Jul (1)	Aug (10)	Sep (36)	Oct (339)	Nov (103)	Dec (152)
2006	Jan (141)	Feb (102)	Mar (125)	Apr (203)	May (57)	Jun (30)	Jul (139)	Aug (46)	Sep (64)	Oct (105)	Nov (34)	Dec (162)
2007	Jan (81)	Feb (57)	Mar (141)	Apr (72)	May (9)	Jun (1)	Jul (144)	Aug (88)	Sep (40)	Oct (43)	Nov (34)	Dec (20)
2008	Jan (44)	Feb (45)	Mar (16)	Apr (36)	May (8)	Jun (77)	Jul (177)	Aug (66)	Sep (8)	Oct (33)	Nov (13)	Dec (37)
2009	Jan (2)	Feb (5)	Mar (8)	Apr	May (36)	Jun (19)	Jul (46)	Aug (8)	Sep (1)	Oct (66)	Nov (61)	Dec (10)
2010	Jan (13)	Feb (16)	Mar (38)	Apr (76)	May (47)	Jun (32)	Jul (35)	Aug (45)	Sep (20)	Oct (61)	Nov (24)	Dec (16)
2011	Jan (22)	Feb (34)	Mar (11)	Apr (8)	May (24)	Jun (23)	Jul (11)	Aug (42)	Sep (81)	Oct (48)	Nov (21)	Dec (20)
2012	Jan (30)	Feb (25)	Mar (4)	Apr (6)	May (1)	Jun (5)	Jul (5)	Aug (8)	Sep (6)	Oct (6)	Nov	Dec

archive-access-cvs Mailing List for Web Archive Access Utilities (Page 10)

archive-access-cvs — CVS commits