|
From: <go...@us...> - 2003-09-06 01:43:13
|
Update of /cvsroot/archive-crawler/ArchiveOpenCrawler/src/org/archive/crawler/basic
In directory sc8-pr-cvs1:/tmp/cvs-serv14925/src/org/archive/crawler/basic
Modified Files:
FetcherHTTPSimple.java
Log Message:
share single httpclient instance, using multi connection manager: risk of sync issues, but benefit of single cookie space
Index: FetcherHTTPSimple.java
===================================================================
RCS file: /cvsroot/archive-crawler/ArchiveOpenCrawler/src/org/archive/crawler/basic/FetcherHTTPSimple.java,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** FetcherHTTPSimple.java 6 Aug 2003 01:19:43 -0000 1.6
--- FetcherHTTPSimple.java 6 Sep 2003 01:43:07 -0000 1.7
***************
*** 13,16 ****
--- 13,17 ----
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
+ import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.cookie.CookiePolicy;
import org.apache.commons.httpclient.methods.GetMethod;
***************
*** 18,22 ****
import org.archive.crawler.datamodel.CrawlURI;
import org.archive.crawler.datamodel.FetchStatusCodes;
- import org.archive.crawler.datamodel.InstancePerThread;
import org.archive.crawler.framework.CrawlController;
import org.archive.crawler.framework.Processor;
--- 19,22 ----
***************
*** 29,33 ****
*
*/
! public class FetcherHTTPSimple extends Processor implements InstancePerThread, CoreAttributeConstants, FetchStatusCodes {
private static String XP_TIMEOUT_SECONDS = "//params/@timeout-seconds";
private static int DEFAULT_TIMEOUT_SECONDS = 10;
--- 29,35 ----
*
*/
! public class FetcherHTTPSimple
! extends Processor
! implements CoreAttributeConstants, FetchStatusCodes {
private static String XP_TIMEOUT_SECONDS = "//params/@timeout-seconds";
private static int DEFAULT_TIMEOUT_SECONDS = 10;
***************
*** 124,127 ****
--- 126,130 ----
} finally {
//controller.getKicker().cancelKick(Thread.currentThread());
+ get.releaseConnection();
}
}
***************
*** 134,138 ****
timeout = 1000*getIntAt(XP_TIMEOUT_SECONDS, DEFAULT_TIMEOUT_SECONDS);
CookiePolicy.setDefaultPolicy(CookiePolicy.COMPATIBILITY);
! http = new HttpClient();
}
--- 137,143 ----
timeout = 1000*getIntAt(XP_TIMEOUT_SECONDS, DEFAULT_TIMEOUT_SECONDS);
CookiePolicy.setDefaultPolicy(CookiePolicy.COMPATIBILITY);
! MultiThreadedHttpConnectionManager connectionManager =
! new MultiThreadedHttpConnectionManager();
! http = new HttpClient(connectionManager);
}
|