archive-access-cvs Mailing List for Web Archive Access Utilities (Page 70)

SourceForge Headquarters 225 Broadway Suite 1600 San Diego, CA 92101 +1 (858) 422-6466

Revision: 2080
          http://archive-access.svn.sourceforge.net/archive-access/?rev=2080&view=rev
Author:   bradtofel
Date:     2007-11-27 16:57:47 -0800 (Tue, 27 Nov 2007)

Log Message:
-----------
UNDO: backed out useless changes for DurableArchiveRecordIterator.. Changes need to be made down in ARCReader -- it does not indicate the end of the record begun when a problem is encountered.

Modified Paths:
--------------
    trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/ArcIndexer.java

Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/ArcIndexer.java
===================================================================

--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/ArcIndexer.java	2007-11-28 00:43:02 UTC (rev 2079)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/ArcIndexer.java	2007-11-28 00:57:47 UTC (rev 2080)
@@ -28,7 +28,6 @@
 import java.io.PrintWriter;
 import java.io.IOException;
 import java.util.Iterator;
-import java.util.NoSuchElementException;
 
 import org.archive.io.ArchiveRecord;
 import org.archive.io.arc.ARCReader;
@@ -61,8 +60,8 @@
 	 */
 	public CloseableIterator<SearchResult> iterator(File arc)
 	throws IOException {
-//		ARCReader arcReader = ARCReaderFactory.get(arc);
-//		arcReader.setParseHttpHeaders(true);
+		ARCReader arcReader = ARCReaderFactory.get(arc);
+		arcReader.setParseHttpHeaders(true);
 
 		Adapter<ArchiveRecord,ARCRecord> adapter1 =
 			new ArchiveRecordToARCRecordAdapter();
@@ -70,7 +69,7 @@
 		Adapter<ARCRecord,SearchResult> adapter2 =
 			new ARCRecordToSearchResultAdapter();
 		
-		Iterator<ArchiveRecord> itr1 = new DurableArchiveRecordIterator(arc);
+		Iterator<ArchiveRecord> itr1 = arcReader.iterator();
 
 		CloseableIterator<ARCRecord> itr2 = 
 			new AdaptedIterator<ArchiveRecord,ARCRecord>(itr1,adapter1);
@@ -94,86 +93,86 @@
 		}
 	}
 	
-	private class DurableArchiveRecordIterator 
-	implements Iterator<ArchiveRecord> {
+//	private class DurableArchiveRecordIterator 
+//	implements Iterator<ArchiveRecord> {
+//
+//		private long lastRestart = 0;
+//		private File arc = null;
+//		Iterator<ArchiveRecord> innerItr = null;
+//		ArchiveRecord cachedNext = null;
+//		
+//		public DurableArchiveRecordIterator(File arc) throws IOException {
+//			this.arc = arc;
+//			restart(0);
+//		}
+//
+//		private void restart(long offset) throws IOException {
+//			ARCReader arcReader = ARCReaderFactory.get(arc,offset);
+//			arcReader.setParseHttpHeaders(true);
+//			innerItr = arcReader.iterator();
+//		}
+//		
+//		private long parseErrorOffset(String message) {
+//			long found = -1;
+//			int idx = message.indexOf("Offset ");
+//			if(idx >= 0) {
+//				int idx2 = message.indexOf(")");
+//				if(idx2 > 0) {
+//					String part = message.substring(idx + 7,idx2);
+//					System.err.println("Found(" + part +") from (" + message + ")");
+//					found = Long.parseLong(part) + 100;
+//				}
+//			}
+//			return found;
+//		}
+//		
+//		public boolean hasNext() {
+//			if(cachedNext != null) {
+//				return true;
+//			}
+//			while(true) {
+//				try {
+//					if(!innerItr.hasNext()) {
+//						return false;
+//					}
+//					cachedNext = innerItr.next();
+//				} catch (RuntimeException e) {
+//					long offset = parseErrorOffset(e.getMessage());
+//					if(offset > 0) {
+//						if(lastRestart == offset) {
+//							return false;
+//						}
+//						lastRestart = offset;
+//						try {
+//							restart(offset);
+//						} catch (IOException e1) {
+//							throw new RuntimeException(e1);
+//						}
+//					} else {
+//						throw e;
+//					}
+//				}
+//				if(cachedNext != null) {
+//					break;
+//				}
+//			}
+//			return true;
+//		}
+//
+//		public ArchiveRecord next() {
+//			if(cachedNext == null) {
+//				throw new NoSuchElementException("next() without hasNext()");
+//			}
+//			ArchiveRecord tmp = cachedNext;
+//			cachedNext = null;
+//			return tmp;
+//		}
+//
+//		public void remove() {
+//			throw new UnsupportedOperationException("remove unimplemented");
+//		}
+//	}
 
-		private long lastRestart = 0;
-		private File arc = null;
-		Iterator<ArchiveRecord> innerItr = null;
-		ArchiveRecord cachedNext = null;
-		
-		public DurableArchiveRecordIterator(File arc) throws IOException {
-			this.arc = arc;
-			restart(0);
-		}
-
-		private void restart(long offset) throws IOException {
-			ARCReader arcReader = ARCReaderFactory.get(arc,offset);
-			arcReader.setParseHttpHeaders(true);
-			innerItr = arcReader.iterator();
-		}
-		
-		private long parseErrorOffset(String message) {
-			long found = -1;
-			int idx = message.indexOf("Offset ");
-			if(idx >= 0) {
-				int idx2 = message.indexOf(")");
-				if(idx2 > 0) {
-					String part = message.substring(idx + 7,idx2);
-					System.err.println("Found(" + part +") from (" + message + ")");
-					found = Long.parseLong(part) + 100;
-				}
-			}
-			return found;
-		}
-		
-		public boolean hasNext() {
-			if(cachedNext != null) {
-				return true;
-			}
-			while(true) {
-				try {
-					if(!innerItr.hasNext()) {
-						return false;
-					}
-					cachedNext = innerItr.next();
-				} catch (RuntimeException e) {
-					long offset = parseErrorOffset(e.getMessage());
-					if(offset > 0) {
-						if(lastRestart == offset) {
-							return false;
-						}
-						lastRestart = offset;
-						try {
-							restart(offset);
-						} catch (IOException e1) {
-							throw new RuntimeException(e1);
-						}
-					} else {
-						throw e;
-					}
-				}
-				if(cachedNext != null) {
-					break;
-				}
-			}
-			return true;
-		}
-
-		public ArchiveRecord next() {
-			if(cachedNext == null) {
-				throw new NoSuchElementException("next() without hasNext()");
-			}
-			ArchiveRecord tmp = cachedNext;
-			cachedNext = null;
-			return tmp;
-		}
-
-		public void remove() {
-			throw new UnsupportedOperationException("remove unimplemented");
-		}
-	}
-
 	private static void USAGE() {
 		System.err.println("USAGE:");
 		System.err.println("");


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




2005	Jan	Feb	Mar	Apr	May	Jun	Jul (1)	Aug (10)	Sep (36)	Oct (339)	Nov (103)	Dec (152)
2006	Jan (141)	Feb (102)	Mar (125)	Apr (203)	May (57)	Jun (30)	Jul (139)	Aug (46)	Sep (64)	Oct (105)	Nov (34)	Dec (162)
2007	Jan (81)	Feb (57)	Mar (141)	Apr (72)	May (9)	Jun (1)	Jul (144)	Aug (88)	Sep (40)	Oct (43)	Nov (34)	Dec (20)
2008	Jan (44)	Feb (45)	Mar (16)	Apr (36)	May (8)	Jun (77)	Jul (177)	Aug (66)	Sep (8)	Oct (33)	Nov (13)	Dec (37)
2009	Jan (2)	Feb (5)	Mar (8)	Apr	May (36)	Jun (19)	Jul (46)	Aug (8)	Sep (1)	Oct (66)	Nov (61)	Dec (10)
2010	Jan (13)	Feb (16)	Mar (38)	Apr (76)	May (47)	Jun (32)	Jul (35)	Aug (45)	Sep (20)	Oct (61)	Nov (24)	Dec (16)
2011	Jan (22)	Feb (34)	Mar (11)	Apr (8)	May (24)	Jun (23)	Jul (11)	Aug (42)	Sep (81)	Oct (48)	Nov (21)	Dec (20)
2012	Jan (30)	Feb (25)	Mar (4)	Apr (6)	May (1)	Jun (5)	Jul (5)	Aug (8)	Sep (6)	Oct (6)	Nov	Dec

archive-access-cvs Mailing List for Web Archive Access Utilities (Page 70)

archive-access-cvs — CVS commits