Menu
▾
▴
archive-access-cvs
Revision: 3387
http://archive-access.svn.sourceforge.net/archive-access/?rev=3387&view=rev
Author: bradtofel
Date: 2011-02-06 14:31:04 +0000 (Sun, 06 Feb 2011)
Log Message:
-----------
added tests for real-world examples
Modified Paths:
--------------
trunk/archive-access/projects/wayback/wayback-core/src/test/java/org/archive/wayback/accesscontrol/staticmap/StaticMapExclusionFilterTest.java
Modified: trunk/archive-access/projects/wayback/wayback-core/src/test/java/org/archive/wayback/accesscontrol/staticmap/StaticMapExclusionFilterTest.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/test/java/org/archive/wayback/accesscontrol/staticmap/StaticMapExclusionFilterTest.java 2011-01-19 03:28:43 UTC (rev 3386)
+++ trunk/archive-access/projects/wayback/wayback-core/src/test/java/org/archive/wayback/accesscontrol/staticmap/StaticMapExclusionFilterTest.java 2011-02-06 14:31:04 UTC (rev 3387)
@@ -24,8 +24,10 @@
import java.io.IOException;
import java.util.Map;
+import org.archive.wayback.UrlCanonicalizer;
import org.archive.wayback.core.CaptureSearchResult;
import org.archive.wayback.util.ObjectFilter;
+import org.archive.wayback.util.url.AggressiveUrlCanonicalizer;
import junit.framework.TestCase;
@@ -39,6 +41,7 @@
File tmpFile = null;
StaticMapExclusionFilterFactory factory = null;
+ UrlCanonicalizer canonicalizer = new AggressiveUrlCanonicalizer();
protected void setUp() throws Exception {
super.setUp();
@@ -62,6 +65,71 @@
/**
* @throws Exception
*/
+ public void testRealWorld() throws Exception {
+ String bases[] = { "pho-c.co.jp/~clever",
+ "sf.net/pop/Roger",
+ "www.eva-stu.vn",
+ "mins.com.br/",
+ "24.ne.jp",
+ "24.ne.jp/~nekko"};
+// setTmpContents(bases);
+
+
+ ObjectFilter<CaptureSearchResult> filter = getFilter(bases);
+ assertFalse("unmassaged",isBlocked(filter,"24.ne.jp.idpnt.com/robots.txt"));
+ assertTrue("massage",isBlocked(filter,"http://24.ne.jp:80/"));
+ assertTrue("unmassaged",isBlocked(filter,"http://www.pho-c.co.jp/~clever"));
+ assertTrue("massage",isBlocked(filter,"http://24.ne.jp"));
+
+
+ assertTrue("unmassaged",isBlocked(filter,"http://www.pho-c.co.jp/~clever"));
+ assertTrue("massaged",isBlocked(filter,"http://pho-c.co.jp/~clever"));
+ assertTrue("trailing-slash",isBlocked(filter,"http://pho-c.co.jp/~clever/"));
+ assertTrue("subpath",isBlocked(filter,"http://pho-c.co.jp/~clever/foo.txt"));
+
+ assertTrue("full-port",isBlocked(filter,"http://www.mins.com.br:80"));
+ assertTrue("tail-slash-port",isBlocked(filter,"http://www.mins.com.br:80/"));
+ assertTrue("full",isBlocked(filter,"http://www.mins.com.br"));
+ assertTrue("tail-slash",isBlocked(filter,"http://www.mins.com.br/"));
+ assertTrue("full-massage",isBlocked(filter,"http://mins.com.br"));
+ assertTrue("tail-slash-massage",isBlocked(filter,"http://mins.com.br/"));
+ assertTrue("massage",isBlocked(filter,"http://mins.com.br/foo.txt"));
+ assertTrue("subpath",isBlocked(filter,"http://www13.mins.com.br/~clever/foo.txt"));
+
+ assertTrue("massage",isBlocked(filter,"24.ne.jp"));
+ assertTrue("full",isBlocked(filter,"http://www.mins.com.br"));
+ assertTrue("subpath",isBlocked(filter,"www.24.ne.jp"));
+ assertTrue("tail-slash-massage",isBlocked(filter,"http://mins.com.br/"));
+ assertTrue("subpath",isBlocked(filter,"http://www.24.ne.jp:80/"));
+
+
+
+
+ assertTrue(isBlocked(filter,"http://sf.net/pop/Roger"));
+ assertTrue(isBlocked(filter,"http://sf.net/pop/Roger/"));
+ assertTrue(isBlocked(filter,"http://sf.net/pop/Roger//"));
+ assertFalse(isBlocked(filter,"http://sf.net/pop/"));
+ assertTrue(isBlocked(filter,"http://sf.net/pop/Roger/2"));
+ assertTrue(isBlocked(filter,"http://sf.net/pop/Roger/23"));
+ assertTrue(isBlocked(filter,"http://www.sf.net/pop/Roger"));
+ assertTrue(isBlocked(filter,"http://www1.sf.net/pop/Roger"));
+ assertTrue(isBlocked(filter,"http://www23.sf.net/pop/Roger"));
+
+ assertTrue(isBlocked(filter,"http://www23.eva-stu.vn/"));
+ assertTrue(isBlocked(filter,"http://www23.eva-stu.vn"));
+ assertTrue(isBlocked(filter,"http://eva-stu.vn"));
+ assertTrue(isBlocked(filter,"http://www.eva-stu.vn/"));
+ assertTrue(isBlocked(filter,"http://eva-stu.vn/"));
+ assertTrue(isBlocked(filter,"http://www.eva-stu.vn/foo.txt"));
+ assertTrue(isBlocked(filter,"http://www2.eva-stu.vn/foo/bar.txt"));
+ assertTrue(isBlocked(filter,"http://eva-stu.vn/foo/bar.txt"));
+
+ }
+
+
+ /**
+ * @throws Exception
+ */
public void testBaseNoPrefix() throws Exception {
String bases[] = {"http://www.peagreenboat.com/",
"http://peagreenboat.com/"};
@@ -93,7 +161,7 @@
setTmpContents(lines);
Map<String,Object> map = factory.loadFile(tmpFile.getAbsolutePath());
- return new StaticMapExclusionFilter(map);
+ return new StaticMapExclusionFilter(map,canonicalizer);
}
private void setTmpContents(String[] lines) throws IOException {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
Revision: 3488
http://archive-access.svn.sourceforge.net/archive-access/?rev=3488&view=rev
Author: bradtofel
Date: 2011-07-08 05:13:03 +0000 (Fri, 08 Jul 2011)
Log Message:
-----------
Modified Paths:
--------------
trunk/archive-access/projects/wayback/wayback-core/src/test/java/org/archive/wayback/accesscontrol/staticmap/StaticMapExclusionFilterTest.java
Modified: trunk/archive-access/projects/wayback/wayback-core/src/test/java/org/archive/wayback/accesscontrol/staticmap/StaticMapExclusionFilterTest.java
===================================================================
--- trunk/archive-access/projects/wayback/wayback-core/src/test/java/org/archive/wayback/accesscontrol/staticmap/StaticMapExclusionFilterTest.java 2011-07-08 04:50:15 UTC (rev 3487)
+++ trunk/archive-access/projects/wayback/wayback-core/src/test/java/org/archive/wayback/accesscontrol/staticmap/StaticMapExclusionFilterTest.java 2011-07-08 05:13:03 UTC (rev 3488)
@@ -24,6 +24,7 @@
import java.io.IOException;
import java.util.Map;
+import org.apache.commons.httpclient.URIException;
import org.archive.wayback.UrlCanonicalizer;
import org.archive.wayback.core.CaptureSearchResult;
import org.archive.wayback.util.ObjectFilter;
@@ -146,9 +147,10 @@
assertTrue("emptypath",isBlocked(filter,"http://www.peagreenboat.com/"));
}
- private boolean isBlocked(ObjectFilter<CaptureSearchResult> filter, String url) {
+ private boolean isBlocked(ObjectFilter<CaptureSearchResult> filter, String url) throws URIException {
CaptureSearchResult result = new CaptureSearchResult();
result.setOriginalUrl(url);
+ result.setUrlKey(canonicalizer.urlStringToKey(url));
int filterResult = filter.filterObject(result);
if(filterResult == ObjectFilter.FILTER_EXCLUDE) {
return true;
@@ -163,7 +165,7 @@
Map<String,Object> map = factory.loadFile(tmpFile.getAbsolutePath());
return new StaticMapExclusionFilter(map,canonicalizer);
}
-
+
private void setTmpContents(String[] lines) throws IOException {
if(tmpFile != null && tmpFile.exists()) {
tmpFile.delete();
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
Thanks for helping keep SourceForge clean.
X