Menu
▾
▴
[Archive-access-cvs] SF.net SVN: archive-access: [1926] trunk/archive-access/projects/wayback/ wayback-core/src/main/java/org/archive/wayback
|
From: <bra...@us...> - 2007-08-23 21:33:51
|
Revision: 1926
http://archive-access.svn.sourceforge.net/archive-access/?rev=1926&view=rev
Author: bradtofel
Date: 2007-08-23 14:33:51 -0700 (Thu, 23 Aug 2007)
Log Message:
-----------
INTERFACE: get() now accepts a WaybackRequest object, so the particular filters returned can depend on user request info: source IP, for example.
Modified Paths:
--------------
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/CompositeExclusionFilterFactory.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionFilterFactory.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/remote/RemoteExclusionFilterFactory.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/robotstxt/RobotExclusionFilterFactory.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/staticmap/StaticMapExclusionFilterFactory.java
trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/LocalResourceIndex.java
Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/CompositeExclusionFilterFactory.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/CompositeExclusionFilterFactory.java 2007-08-23 21:32:24 UTC (rev 1925)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/CompositeExclusionFilterFactory.java 2007-08-23 21:33:51 UTC (rev 1926)
@@ -28,6 +28,7 @@
import java.util.Iterator;
import org.archive.wayback.core.SearchResult;
+import org.archive.wayback.core.WaybackRequest;
import org.archive.wayback.resourceindex.filters.CompositeExclusionFilter;
import org.archive.wayback.util.ObjectFilter;
@@ -54,11 +55,11 @@
/* (non-Javadoc)
* @see org.archive.wayback.resourceindex.ExclusionFilterFactory#get()
*/
- public ObjectFilter<SearchResult> get() {
+ public ObjectFilter<SearchResult> get(WaybackRequest wbRequest) {
Iterator<ExclusionFilterFactory> itr = factories.iterator();
CompositeExclusionFilter filter = new CompositeExclusionFilter();
while(itr.hasNext()) {
- filter.addComponent(itr.next().get());
+ filter.addComponent(itr.next().get(wbRequest));
}
return filter;
}
@@ -78,4 +79,15 @@
public void setFactories(ArrayList<ExclusionFilterFactory> factories) {
this.factories = factories;
}
+
+ /* (non-Javadoc)
+ * @see org.archive.wayback.accesscontrol.ExclusionFilterFactory#shutdown()
+ */
+ public void shutdown() {
+ Iterator<ExclusionFilterFactory> itr = factories.iterator();
+ while(itr.hasNext()) {
+ ExclusionFilterFactory i = itr.next();
+ i.shutdown();
+ }
+ }
}
Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionFilterFactory.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionFilterFactory.java 2007-08-23 21:32:24 UTC (rev 1925)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/ExclusionFilterFactory.java 2007-08-23 21:33:51 UTC (rev 1926)
@@ -25,6 +25,7 @@
package org.archive.wayback.accesscontrol;
import org.archive.wayback.core.SearchResult;
+import org.archive.wayback.core.WaybackRequest;
import org.archive.wayback.util.ObjectFilter;
/**
*
@@ -34,8 +35,13 @@
*/
public interface ExclusionFilterFactory {
/**
+ * @param wbRequest
* @return an ObjectFilter object that filters records based on
* some set of exclusion rules.
*/
- public ObjectFilter<SearchResult> get();
+ public ObjectFilter<SearchResult> get(WaybackRequest wbRequest);
+ /**
+ * close any resources used by this ExclusionFilter system.
+ */
+ public void shutdown();
}
Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/remote/RemoteExclusionFilterFactory.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/remote/RemoteExclusionFilterFactory.java 2007-08-23 21:32:24 UTC (rev 1925)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/remote/RemoteExclusionFilterFactory.java 2007-08-23 21:33:51 UTC (rev 1926)
@@ -26,6 +26,7 @@
import org.archive.wayback.accesscontrol.ExclusionFilterFactory;
import org.archive.wayback.core.SearchResult;
+import org.archive.wayback.core.WaybackRequest;
import org.archive.wayback.util.ObjectFilter;
/**
@@ -43,7 +44,7 @@
/* (non-Javadoc)
* @see org.archive.wayback.resourceindex.ExclusionFilterFactory#get()
*/
- public ObjectFilter<SearchResult> get() {
+ public ObjectFilter<SearchResult> get(WaybackRequest wbRequest) {
return new RemoteExclusionFilter(exclusionUrlPrefix, exclusionUserAgent);
}
@@ -75,5 +76,12 @@
this.exclusionUserAgent = exclusionUserAgent;
}
+ /* (non-Javadoc)
+ * @see org.archive.wayback.accesscontrol.ExclusionFilterFactory#shutdown()
+ */
+ public void shutdown() {
+ // nothing to do..
+ }
+
}
Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/robotstxt/RobotExclusionFilterFactory.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/robotstxt/RobotExclusionFilterFactory.java 2007-08-23 21:32:24 UTC (rev 1925)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/robotstxt/RobotExclusionFilterFactory.java 2007-08-23 21:33:51 UTC (rev 1926)
@@ -26,6 +26,7 @@
import org.archive.wayback.accesscontrol.ExclusionFilterFactory;
import org.archive.wayback.core.SearchResult;
+import org.archive.wayback.core.WaybackRequest;
import org.archive.wayback.liveweb.LiveWebCache;
import org.archive.wayback.util.ObjectFilter;
@@ -44,7 +45,7 @@
/* (non-Javadoc)
* @see org.archive.wayback.resourceindex.ExclusionFilterFactory#get()
*/
- public ObjectFilter<SearchResult> get() {
+ public ObjectFilter<SearchResult> get(WaybackRequest wbRequest) {
return new RobotExclusionFilter(webCache,userAgent,maxCacheMS);
}
@@ -89,4 +90,11 @@
public void setMaxCacheMS(long maxCacheMS) {
this.maxCacheMS = maxCacheMS;
}
+
+ /* (non-Javadoc)
+ * @see org.archive.wayback.accesscontrol.ExclusionFilterFactory#shutdown()
+ */
+ public void shutdown() {
+ webCache.shutdown();
+ }
}
Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/staticmap/StaticMapExclusionFilterFactory.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/staticmap/StaticMapExclusionFilterFactory.java 2007-08-23 21:32:24 UTC (rev 1925)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/accesscontrol/staticmap/StaticMapExclusionFilterFactory.java 2007-08-23 21:33:51 UTC (rev 1926)
@@ -32,6 +32,7 @@
import org.archive.wayback.accesscontrol.ExclusionFilterFactory;
import org.archive.wayback.core.SearchResult;
+import org.archive.wayback.core.WaybackRequest;
import org.archive.wayback.surt.SURTTokenizer;
import org.archive.wayback.util.CloseableIterator;
import org.archive.wayback.util.ObjectFilter;
@@ -47,7 +48,7 @@
private static final Logger LOGGER =
Logger.getLogger(StaticMapExclusionFilterFactory.class.getName());
- private final static int checkInterval = 10;
+ private int checkInterval = 10;
private Map<String,Object> currentMap = null;
private File file = null;
long lastUpdated = 0;
@@ -88,6 +89,10 @@
CloseableIterator<String> itr = ff.getSequentialIterator();
while(itr.hasNext()) {
String line = (String) itr.next();
+ line = line.trim();
+ if(line.length() == 0) {
+ continue;
+ }
String surt = line.startsWith("(") ? line :
SURTTokenizer.prefixKey(line);
newMap.put(surt, null);
@@ -97,9 +102,10 @@
}
/**
+ * @param wbRequest
* @return SearchResultFilter
*/
- public ObjectFilter<SearchResult> get() {
+ public ObjectFilter<SearchResult> get(WaybackRequest wbRequest) {
if(currentMap == null) {
return null;
}
@@ -113,6 +119,12 @@
updateThread = new CacheUpdaterThread(this,checkInterval);
updateThread.start();
}
+ private synchronized void stopUpdateThread() {
+ if (updateThread == null) {
+ return;
+ }
+ updateThread.interrupt();
+ }
private class CacheUpdaterThread extends Thread {
/**
@@ -147,8 +159,45 @@
Thread.sleep(sleepInterval * 1000);
} catch (InterruptedException e) {
e.printStackTrace();
+ return;
}
}
}
}
+
+ /**
+ * @return the checkInterval
+ */
+ public int getCheckInterval() {
+ return checkInterval;
+ }
+
+ /**
+ * @param checkInterval the checkInterval to set
+ */
+ public void setCheckInterval(int checkInterval) {
+ this.checkInterval = checkInterval;
+ }
+
+ /**
+ * @return the path
+ */
+ public String getFile() {
+ return file.getAbsolutePath();
+ }
+
+ /**
+ * @param path the file to set
+ */
+ public void setFile(String path) {
+ this.file = new File(path);
+ }
+
+ /* (non-Javadoc)
+ * @see org.archive.wayback.accesscontrol.ExclusionFilterFactory#shutdown()
+ */
+ public void shutdown() {
+ // TODO Auto-generated method stub
+ stopUpdateThread();
+ }
}
Modified: trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/LocalResourceIndex.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/LocalResourceIndex.java 2007-08-23 21:32:24 UTC (rev 1925)
+++ trunk/archive-access/projects/wayback/wayback-core/src/main/java/org/archive/wayback/resourceindex/LocalResourceIndex.java 2007-08-23 21:33:51 UTC (rev 1926)
@@ -80,11 +80,12 @@
private UrlCanonicalizer canonicalizer = new UrlCanonicalizer();
- private ObjectFilter<SearchResult> getExclusionFilter()
+ private ObjectFilter<SearchResult> getExclusionFilter(
+ WaybackRequest wbRequest)
throws ResourceIndexNotAvailableException {
ObjectFilter<SearchResult> filter = null;
if(exclusionFactory != null) {
- filter = exclusionFactory.get();
+ filter = exclusionFactory.get(wbRequest);
if(filter == null) {
throw new ResourceIndexNotAvailableException("Exclusion " +
"Service Unavailable");
@@ -210,7 +211,7 @@
GuardRailFilter guardrail = new GuardRailFilter(maxRecords);
// checks an exclusion service for every matching record
- ObjectFilter<SearchResult> exclusion = getExclusionFilter();
+ ObjectFilter<SearchResult> exclusion = getExclusionFilter(wbRequest);
// count how many results got to the ExclusionFilter:
CounterFilter preExCounter = new CounterFilter();
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
