Menu
▾
▴
[Archive-access-cvs] SF.net SVN: archive-access: [2209] trunk/archive-access/projects/wayback/ wayback-core/src/main/java/org/archive/wayback/resourcestore
|
From: <bra...@us...> - 2008-03-01 01:59:20
|
Revision: 2209
http://archive-access.svn.sourceforge.net/archive-access/?rev=2209&view=rev
Author: bradtofel
Date: 2008-02-29 17:59:24 -0800 (Fri, 29 Feb 2008)
Log Message:
-----------
BUGFIX (ACC-12): LocalResourceStore now returns a CloseableIterator to AutoIndexThread, which now calls close after indexing.
Modified Paths:
--------------
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/ArcIndexer.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/AutoIndexThread.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/LocalResourceStore.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/WarcIndexer.java
Added Paths:
-----------
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/ArchiveReaderCloseableIterator.java
Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/ArcIndexer.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/ArcIndexer.java 2008-02-27 11:28:06 UTC (rev 2208)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/ArcIndexer.java 2008-03-01 01:59:24 UTC (rev 2209)
@@ -77,7 +77,8 @@
new ARCRecordToSearchResultAdapter();
adapter2.setCanonicalizer(canonicalizer);
- Iterator<ArchiveRecord> itr1 = arcReader.iterator();
+ ArchiveReaderCloseableIterator itr1 =
+ new ArchiveReaderCloseableIterator(arcReader,arcReader.iterator());
CloseableIterator<ARCRecord> itr2 =
new AdaptedIterator<ArchiveRecord,ARCRecord>(itr1,adapter1);
Added: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/ArchiveReaderCloseableIterator.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/ArchiveReaderCloseableIterator.java (rev 0)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/ArchiveReaderCloseableIterator.java 2008-03-01 01:59:24 UTC (rev 2209)
@@ -0,0 +1,29 @@
+package org.archive.wayback.resourcestore;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.archive.io.ArchiveReader;
+import org.archive.io.ArchiveRecord;
+import org.archive.wayback.util.CloseableIterator;
+
+public class ArchiveReaderCloseableIterator implements CloseableIterator<ArchiveRecord> {
+ private ArchiveReader reader = null;
+ private Iterator<ArchiveRecord> itr = null;
+ public ArchiveReaderCloseableIterator(ArchiveReader reader, Iterator<ArchiveRecord> itr) {
+ this.reader = reader;
+ this.itr = itr;
+ }
+ public boolean hasNext() {
+ return itr.hasNext();
+ }
+ public ArchiveRecord next() {
+ return itr.next();
+ }
+ public void remove() {
+ itr.remove();
+ }
+ public void close() throws IOException {
+ reader.close();
+ }
+}
Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/AutoIndexThread.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/AutoIndexThread.java 2008-02-27 11:28:06 UTC (rev 2208)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/AutoIndexThread.java 2008-03-01 01:59:24 UTC (rev 2209)
@@ -9,6 +9,7 @@
import org.archive.wayback.core.SearchResult;
import org.archive.wayback.resourceindex.indexer.IndexClient;
+import org.archive.wayback.util.CloseableIterator;
import org.archive.wayback.util.DirMaker;
/**
@@ -150,7 +151,7 @@
try {
LOGGER.info("Indexing " + file.getAbsolutePath());
- Iterator<SearchResult> itr = store.indexFile(file);
+ CloseableIterator<SearchResult> itr = store.indexFile(file);
if(indexClient.addSearchResults(cdxBase, itr)) {
if (!workFlagFile.delete()) {
@@ -158,6 +159,7 @@
+ workFlagFile.getAbsolutePath());
}
}
+ itr.close();
numIndexed++;
} catch (IOException e) {
LOGGER.severe("FAILED index: " + file.getAbsolutePath()
Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/LocalResourceStore.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/LocalResourceStore.java 2008-02-27 11:28:06 UTC (rev 2208)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/LocalResourceStore.java 2008-03-01 01:59:24 UTC (rev 2209)
@@ -13,6 +13,7 @@
import org.archive.wayback.core.SearchResult;
import org.archive.wayback.exception.ConfigurationException;
import org.archive.wayback.exception.ResourceNotAvailableException;
+import org.archive.wayback.util.CloseableIterator;
import org.archive.wayback.util.DirMaker;
/**
@@ -88,8 +89,8 @@
}
}
- public Iterator<SearchResult> indexFile(File dataFile) throws IOException {
- Iterator<SearchResult> itr = null;
+ public CloseableIterator<SearchResult> indexFile(File dataFile) throws IOException {
+ CloseableIterator<SearchResult> itr = null;
String name = dataFile.getName();
if(name.endsWith(ARC_EXTENSION)) {
@@ -140,4 +141,7 @@
public void setIndexThread(AutoIndexThread indexThread) {
this.indexThread = indexThread;
}
+ public void shutdown() throws IOException {
+ // no-op. could shut down threads
+ }
}
Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/WarcIndexer.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/WarcIndexer.java 2008-02-27 11:28:06 UTC (rev 2208)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourcestore/WarcIndexer.java 2008-03-01 01:59:24 UTC (rev 2209)
@@ -46,10 +46,11 @@
WARCReader reader = WARCReaderFactory.get(warc);
- Iterator<ArchiveRecord> itr1 = reader.iterator();
+ ArchiveReaderCloseableIterator itr1 =
+ new ArchiveReaderCloseableIterator(reader,reader.iterator());
- CloseableIterator<WARCRecord> itr2 = new AdaptedIterator<ArchiveRecord, WARCRecord>(
- itr1, adapter1);
+ CloseableIterator<WARCRecord> itr2 =
+ new AdaptedIterator<ArchiveRecord, WARCRecord>(itr1, adapter1);
return new AdaptedIterator<WARCRecord, SearchResult>(itr2, adapter2);
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
