Heritrix: Internet Archive Web Crawler

20041103031304383 -2 -
https://login.yahoo.com/robots.txt LRXRP
https://login.yahoo.com/config/login?.src=chat&.done=http://chat.yahoo.com/
c/roomlist.html&.intl=us
no-type #041 - - -
java.net.SocketException: already connected
at java.net.Socket.connect(Socket.java:433)
at
com.sun.net.ssl.internal.ssl.SSLSocketImpl.connect(DashoA6275)
at
org.archive.crawler.fetcher.HeritrixSSLProtocolSocketFactory.createSocket(H
eritrixSSLProtocolSocketFactory.java:135)
at
org.apache.commons.httpclient.HttpConnection.open(HttpConnection.java:669)
at
org.apache.commons.httpclient.MultiThreadedHttpConnectionManager$HttpConnec
tionAdapter.open(MultiThreadedHttpConnectionManager.java:1328)
at
org.apache.commons.httpclient.HttpMethodDirector.executeWithRetry(HttpMetho
dDirector.java:369)
at
org.apache.commons.httpclient.HttpMethodDirector.executeMethod(HttpMethodDi
rector.java:178)
at
org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:437)

at
org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:324)

at
org.archive.crawler.fetcher.FetchHTTP.innerProcess(FetchHTTP.java:299)
at
org.archive.crawler.framework.Processor.process(Processor.java:102)
at
org.archive.crawler.framework.ToeThread.processCrawlUri(ToeThread.java:255)

at
org.archive.crawler.framework.ToeThread.run(ToeThread.java:131)
20041103031309994 -2 -
https://sec.yimg.com/robots.txt LREEP
https://sec.yimg.com/i/b5/arrow.gif no-type #025 - - -
java.net.SocketException: already connected
at java.net.Socket.connect(Socket.java:433)
at
com.sun.net.ssl.internal.ssl.SSLSocketImpl.connect(DashoA6275)
at
org.archive.crawler.fetcher.HeritrixSSLProtocolSocketFactory.createSocket(H
eritrixSSLProtocolSocketFactory.java:135)
at
org.apache.commons.httpclient.HttpConnection.open(HttpConnection.java:669)
at
org.apache.commons.httpclient.MultiThreadedHttpConnectionManager$HttpConnec
tionAdapter.open(MultiThreadedHttpConnectionManager.java:1328)
at
org.apache.commons.httpclient.HttpMethodDirector.executeWithRetry(HttpMetho
dDirector.java:369)
at
org.apache.commons.httpclient.HttpMethodDirector.executeMethod(HttpMethodDi
rector.java:178)
at
org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:437)

at
org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:324)

at
org.archive.crawler.fetcher.FetchHTTP.innerProcess(FetchHTTP.java:299)
at
org.archive.crawler.framework.Processor.process(Processor.java:102)
at
org.archive.crawler.framework.ToeThread.processCrawlUri(ToeThread.java:255)

at
org.archive.crawler.framework.ToeThread.run(ToeThread.java:131)

Looks like problem in my ssl factory:

public Socket createSocket(String host, int port,
InetAddress localAddress,
int localPort, HttpConnectionParams params)
throws IOException, UnknownHostException {
// Below code is from the
DefaultSSLProtocolSocketFactory#createSocket
// method only it has workarounds to deal with
pre-1.4 JVMs. I've
// cut these out.
if (params == null) {
throw new
IllegalArgumentException("Parameters may not be null");
}
Socket socket = null;
int timeout = params.getConnectionTimeout();
if (timeout == 0) {
socket = createSocket(host, port,
localAddress, localPort);
} else {
socket = this.sslfactory.createSocket();
InetAddress hostAddress = getHostAddress(host);
InetSocketAddress address = (hostAddress !=
null)?
new InetSocketAddress(hostAddress,
port):
new InetSocketAddress(host, port);
socket.connect(address, timeout);
try {
socket.connect(address, timeout);
} catch (SocketTimeoutException e) {
// Add timeout info. to the exception.
throw new
SocketTimeoutException(e.getMessage() +
": timeout set at " +
Integer.toString(timeout) + "ms.");
}
assert socket.isConnected(): "Socket not
connected " + host;
}
return socket;
}


I need to somehow ask httpclient if it already has a
socket, and if so, resuse it rather than make a new
connection.