From: <rb...@us...> - 2015-07-01 06:44:08
|
Revision: 10818 http://sourceforge.net/p/htmlunit/code/10818 Author: rbri Date: 2015-07-01 06:44:06 +0000 (Wed, 01 Jul 2015) Log Message: ----------- next try to save some memory Modified Paths: -------------- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java Modified: trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java =================================================================== --- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2015-07-01 06:09:43 UTC (rev 10817) +++ trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2015-07-01 06:44:06 UTC (rev 10818) @@ -754,22 +754,22 @@ if (headerNames != null) { for (final String header : headerNames) { if ("Host".equals(header)) { - list.add(new StaticHttpRequestInterceptor(header, host.toString()) { }); + list.add(new HostHeaderHttpRequestInterceptor(host.toString())); } else if ("User-Agent".equals(header)) { - list.add(new StaticHttpRequestInterceptor(header, userAgent) { }); + list.add(new UserAgentHeaderHttpRequestInterceptor(userAgent)); } else if ("Accept".equals(header) && requestHeaders.get(header) != null) { - list.add(new StaticHttpRequestInterceptor(header, requestHeaders.get(header)) { }); + list.add(new AcceptHeaderHttpRequestInterceptor(requestHeaders.get(header))); } else if ("Accept-Language".equals(header) && requestHeaders.get(header) != null) { - list.add(new StaticHttpRequestInterceptor(header, requestHeaders.get(header)) { }); + list.add(new AcceptLanguageHeaderHttpRequestInterceptor(requestHeaders.get(header))); } else if ("Accept-Encoding".equals(header) && requestHeaders.get(header) != null) { - list.add(new StaticHttpRequestInterceptor(header, requestHeaders.get(header)) { }); + list.add(new AcceptEncodingHeaderHttpRequestInterceptor(requestHeaders.get(header))); } else if ("Referer".equals(header) && requestHeaders.get(header) != null) { - list.add(new StaticHttpRequestInterceptor(header, requestHeaders.get(header)) { }); + list.add(new RefererHeaderHttpRequestInterceptor(requestHeaders.get(header))); } else if ("Connection".equals(header)) { list.add(new RequestClientConnControl()); @@ -778,12 +778,12 @@ list.add(new RequestAddCookies()); } else if ("DNT".equals(header) && webClient_.getOptions().isDoNotTrackEnabled()) { - list.add(new StaticHttpRequestInterceptor(header, "1") { }); + list.add(new DntHeaderHttpRequestInterceptor("1")); } } } else { - list.add(new StaticHttpRequestInterceptor("User-Agent", userAgent) { }); + list.add(new UserAgentHeaderHttpRequestInterceptor(userAgent)); list.add(new RequestAddCookies()); list.add(new RequestClientConnControl()); } @@ -791,7 +791,7 @@ // not all browser versions have DNT by default as part of getHeaderNamesOrdered() // so we add it again, in case if (webClient_.getOptions().isDoNotTrackEnabled()) { - list.add(new StaticHttpRequestInterceptor("DNT", "1") { }); + list.add(new DntHeaderHttpRequestInterceptor("1")); } synchronized (requestHeaders) { @@ -801,27 +801,102 @@ } /** We must have a separate class per header, because of org.apache.http.protocol.ChainBuilder. */ - private abstract static class StaticHttpRequestInterceptor implements HttpRequestInterceptor { - private String name_; + private static final class HostHeaderHttpRequestInterceptor implements HttpRequestInterceptor { private String value_; - StaticHttpRequestInterceptor(final String name, final String value) { - this.name_ = name; - this.value_ = value; + HostHeaderHttpRequestInterceptor(final String value) { + value_ = value; } @Override - public void process(final HttpRequest request, final HttpContext context) - throws HttpException, IOException { - request.setHeader(name_, value_); + public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { + request.setHeader("Host", value_); } } + private static final class UserAgentHeaderHttpRequestInterceptor implements HttpRequestInterceptor { + private String value_; + + UserAgentHeaderHttpRequestInterceptor(final String value) { + value_ = value; + } + + @Override + public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { + request.setHeader("User-Agent", value_); + } + } + + private static final class AcceptHeaderHttpRequestInterceptor implements HttpRequestInterceptor { + private String value_; + + AcceptHeaderHttpRequestInterceptor(final String value) { + value_ = value; + } + + @Override + public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { + request.setHeader("Accept", value_); + } + } + + private static final class AcceptLanguageHeaderHttpRequestInterceptor implements HttpRequestInterceptor { + private String value_; + + AcceptLanguageHeaderHttpRequestInterceptor(final String value) { + value_ = value; + } + + @Override + public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { + request.setHeader("Accept-Language", value_); + } + } + + private static final class AcceptEncodingHeaderHttpRequestInterceptor implements HttpRequestInterceptor { + private String value_; + + AcceptEncodingHeaderHttpRequestInterceptor(final String value) { + value_ = value; + } + + @Override + public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { + request.setHeader("Accept-Encoding", value_); + } + } + + private static final class RefererHeaderHttpRequestInterceptor implements HttpRequestInterceptor { + private String value_; + + RefererHeaderHttpRequestInterceptor(final String value) { + value_ = value; + } + + @Override + public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { + request.setHeader("Referer", value_); + } + } + + private static final class DntHeaderHttpRequestInterceptor implements HttpRequestInterceptor { + private String value_; + + DntHeaderHttpRequestInterceptor(final String value) { + value_ = value; + } + + @Override + public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { + request.setHeader("DNT", value_); + } + } + private static class MultiHttpRequestInterceptor implements HttpRequestInterceptor { private final Map<String, String> map_; MultiHttpRequestInterceptor(final Map<String, String> map) { - this.map_ = map; + map_ = map; } @Override |
From: <asa...@us...> - 2015-10-20 08:20:02
|
Revision: 11426 http://sourceforge.net/p/htmlunit/code/11426 Author: asashour Date: 2015-10-20 08:20:00 +0000 (Tue, 20 Oct 2015) Log Message: ----------- Fix build Modified Paths: -------------- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java Modified: trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java =================================================================== --- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2015-10-20 07:33:29 UTC (rev 11425) +++ trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2015-10-20 08:20:00 UTC (rev 11426) @@ -670,15 +670,19 @@ */ protected DownloadedContent downloadResponseBody(final HttpResponse httpResponse) throws IOException { final HttpEntity httpEntity = httpResponse.getEntity(); - final int status = httpResponse.getStatusLine().getStatusCode(); - if (httpEntity == null || ((status <= HttpStatus.SC_SEE_OTHER || status == HttpStatus.SC_TEMPORARY_REDIRECT) - && webClient_.getOptions().isRedirectEnabled())) { + if (httpEntity == null || isRedirect(httpResponse.getStatusLine().getStatusCode())) { return new DownloadedContent.InMemory(new byte[] {}); } return downloadContent(httpEntity.getContent(), webClient_.getOptions().getMaxInMemory()); } + private boolean isRedirect(final int status) { + return ((status >= HttpStatus.SC_MOVED_PERMANENTLY && status <= HttpStatus.SC_SEE_OTHER) + || status == HttpStatus.SC_TEMPORARY_REDIRECT) + && webClient_.getOptions().isRedirectEnabled(); + } + /** * Reads the content of the stream and saves it in memory or on the file system. * @param is the stream to read |
From: <asa...@us...> - 2015-10-20 09:18:17
|
Revision: 11427 http://sourceforge.net/p/htmlunit/code/11427 Author: asashour Date: 2015-10-20 09:18:15 +0000 (Tue, 20 Oct 2015) Log Message: ----------- Fix build Modified Paths: -------------- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java Modified: trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java =================================================================== --- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2015-10-20 08:20:00 UTC (rev 11426) +++ trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2015-10-20 09:18:15 UTC (rev 11427) @@ -19,7 +19,6 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.EOFException; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -53,7 +52,6 @@ import org.apache.http.HttpRequest; import org.apache.http.HttpRequestInterceptor; import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; import org.apache.http.auth.AuthScope; import org.apache.http.auth.Credentials; import org.apache.http.client.CredentialsProvider; @@ -670,19 +668,13 @@ */ protected DownloadedContent downloadResponseBody(final HttpResponse httpResponse) throws IOException { final HttpEntity httpEntity = httpResponse.getEntity(); - if (httpEntity == null || isRedirect(httpResponse.getStatusLine().getStatusCode())) { + if (httpEntity == null) { return new DownloadedContent.InMemory(new byte[] {}); } return downloadContent(httpEntity.getContent(), webClient_.getOptions().getMaxInMemory()); } - private boolean isRedirect(final int status) { - return ((status >= HttpStatus.SC_MOVED_PERMANENTLY && status <= HttpStatus.SC_SEE_OTHER) - || status == HttpStatus.SC_TEMPORARY_REDIRECT) - && webClient_.getOptions().isRedirectEnabled(); - } - /** * Reads the content of the stream and saves it in memory or on the file system. * @param is the stream to read @@ -717,10 +709,9 @@ LOG.warn("Connection was closed while reading from stream.", e); return new DownloadedContent.InMemory(bos.toByteArray()); } - catch (final EOFException e) { + catch (final IOException e) { // this might happen with broken gzip content - // see com.gargoylesoftware.htmlunit.HttpWebConnection2Test.brokenGzip() - LOG.warn("EndOfFile while reading from stream.", e); + LOG.warn("Exception while reading from stream.", e); return new DownloadedContent.InMemory(bos.toByteArray()); } finally { |
From: <rb...@us...> - 2016-02-07 13:22:28
|
Revision: 11822 http://sourceforge.net/p/htmlunit/code/11822 Author: rbri Date: 2016-02-07 13:22:26 +0000 (Sun, 07 Feb 2016) Log Message: ----------- make checkstyle happy Modified Paths: -------------- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java Modified: trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java =================================================================== --- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2016-02-07 12:41:12 UTC (rev 11821) +++ trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2016-02-07 13:22:26 UTC (rev 11822) @@ -421,7 +421,7 @@ return; } - String filename; + final String filename; if (pairWithFile.getFile() == null) { filename = pairWithFile.getValue(); } @@ -935,7 +935,7 @@ private PoolingHttpClientConnectionManager createConnectionManager(final HttpClientBuilder builder) { final ConnectionSocketFactory socketFactory = new SocksConnectionSocketFactory(); - LayeredConnectionSocketFactory sslSocketFactory; + LayeredConnectionSocketFactory sslSocketFactory = null; try { sslSocketFactory = (LayeredConnectionSocketFactory) FieldUtils.readDeclaredField(builder, "sslSocketFactory", true); |
From: <asa...@us...> - 2016-02-29 09:28:05
|
Revision: 11936 http://sourceforge.net/p/htmlunit/code/11936 Author: asashour Date: 2016-02-29 09:28:03 +0000 (Mon, 29 Feb 2016) Log Message: ----------- Remove deprecation usage, and update code from HttpComponenets Modified Paths: -------------- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java Modified: trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java =================================================================== --- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2016-02-29 08:41:07 UTC (rev 11935) +++ trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2016-02-29 09:28:03 UTC (rev 11936) @@ -32,6 +32,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLContext; @@ -80,9 +81,13 @@ import org.apache.http.config.ConnectionConfig; import org.apache.http.config.RegistryBuilder; import org.apache.http.config.SocketConfig; +import org.apache.http.conn.DnsResolver; import org.apache.http.conn.socket.ConnectionSocketFactory; import org.apache.http.conn.socket.LayeredConnectionSocketFactory; +import org.apache.http.conn.ssl.DefaultHostnameVerifier; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.conn.util.PublicSuffixMatcher; +import org.apache.http.conn.util.PublicSuffixMatcherLoader; import org.apache.http.cookie.CookieSpecProvider; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; @@ -92,9 +97,11 @@ import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.protocol.HttpContext; import org.apache.http.protocol.HttpProcessorBuilder; +import org.apache.http.protocol.HttpRequestExecutor; import org.apache.http.protocol.RequestContent; import org.apache.http.protocol.RequestTargetHost; import org.apache.http.ssl.SSLContexts; +import org.apache.http.util.TextUtils; import com.gargoylesoftware.htmlunit.httpclient.HtmlUnitCookieSpecProvider; import com.gargoylesoftware.htmlunit.httpclient.HtmlUnitCookieStore; @@ -937,34 +944,42 @@ } /** - * Has the exact logic in HttpClientBuilder, but with the ability to configure - * <code>socketFactory</code>. + * Has the exact logic in {@link HttpClientBuilder#build()} which sets the {@code connManager} part, + * but with the ability to configure {@code socketFactory}. */ private PoolingHttpClientConnectionManager createConnectionManager(final HttpClientBuilder builder) { final ConnectionSocketFactory socketFactory = new SocksConnectionSocketFactory(); - LayeredConnectionSocketFactory sslSocketFactory = null; try { - sslSocketFactory = (LayeredConnectionSocketFactory) - FieldUtils.readDeclaredField(builder, "sslSocketFactory", true); - final SocketConfig defaultSocketConfig = (SocketConfig) - FieldUtils.readDeclaredField(builder, "defaultSocketConfig", true); - final ConnectionConfig defaultConnectionConfig = (ConnectionConfig) - FieldUtils.readDeclaredField(builder, "defaultConnectionConfig", true); - final boolean systemProperties = (Boolean) FieldUtils.readDeclaredField(builder, "systemProperties", true); - final int maxConnTotal = (Integer) FieldUtils.readDeclaredField(builder, "maxConnTotal", true); - final int maxConnPerRoute = (Integer) FieldUtils.readDeclaredField(builder, "maxConnPerRoute", true); - HostnameVerifier hostnameVerifier = (HostnameVerifier) - FieldUtils.readDeclaredField(builder, "hostnameVerifier", true); - final SSLContext sslcontext = (SSLContext) FieldUtils.readDeclaredField(builder, "sslContext", true); + PublicSuffixMatcher publicSuffixMatcher = getField(builder, "publicSuffixMatcher"); + if (publicSuffixMatcher == null) { + publicSuffixMatcher = PublicSuffixMatcherLoader.getDefault(); + } + HttpRequestExecutor requestExec = getField(builder, "requestExec"); + if (requestExec == null) { + requestExec = new HttpRequestExecutor(); + } + + LayeredConnectionSocketFactory sslSocketFactory = getField(builder, "sslSocketFactory"); + final SocketConfig defaultSocketConfig = getField(builder, "defaultSocketConfig"); + final ConnectionConfig defaultConnectionConfig = getField(builder, "defaultConnectionConfig"); + final boolean systemProperties = getField(builder, "systemProperties"); + final int maxConnTotal = getField(builder, "maxConnTotal"); + final int maxConnPerRoute = getField(builder, "maxConnPerRoute"); + HostnameVerifier hostnameVerifier = getField(builder, "hostnameVerifier"); + final SSLContext sslcontext = getField(builder, "sslContext"); + final DnsResolver dnsResolver = getField(builder, "dnsResolver"); + final long connTimeToLive = getField(builder, "connTimeToLive"); + final TimeUnit connTimeToLiveTimeUnit = getField(builder, "connTimeToLiveTimeUnit"); + if (sslSocketFactory == null) { final String[] supportedProtocols = systemProperties - ? StringUtils.split(System.getProperty("https.protocols"), ',') : null; + ? split(System.getProperty("https.protocols")) : null; final String[] supportedCipherSuites = systemProperties - ? StringUtils.split(System.getProperty("https.cipherSuites"), ',') : null; + ? split(System.getProperty("https.cipherSuites")) : null; if (hostnameVerifier == null) { - hostnameVerifier = SSLConnectionSocketFactory.BROWSER_COMPATIBLE_HOSTNAME_VERIFIER; + hostnameVerifier = new DefaultHostnameVerifier(publicSuffixMatcher); } if (sslcontext != null) { sslSocketFactory = new SSLConnectionSocketFactory( @@ -984,36 +999,53 @@ } } - final PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager( + final PoolingHttpClientConnectionManager poolingmgr = new PoolingHttpClientConnectionManager( RegistryBuilder.<ConnectionSocketFactory>create() .register("http", socketFactory) .register("https", sslSocketFactory) - .build()); + .build(), + null, + null, + dnsResolver, + connTimeToLive, + connTimeToLiveTimeUnit != null ? connTimeToLiveTimeUnit : TimeUnit.MILLISECONDS); if (defaultSocketConfig != null) { - connectionManager.setDefaultSocketConfig(defaultSocketConfig); + poolingmgr.setDefaultSocketConfig(defaultSocketConfig); } if (defaultConnectionConfig != null) { - connectionManager.setDefaultConnectionConfig(defaultConnectionConfig); + poolingmgr.setDefaultConnectionConfig(defaultConnectionConfig); } if (systemProperties) { String s = System.getProperty("http.keepAlive", "true"); if ("true".equalsIgnoreCase(s)) { s = System.getProperty("http.maxConnections", "5"); final int max = Integer.parseInt(s); - connectionManager.setDefaultMaxPerRoute(max); - connectionManager.setMaxTotal(2 * max); + poolingmgr.setDefaultMaxPerRoute(max); + poolingmgr.setMaxTotal(2 * max); } } if (maxConnTotal > 0) { - connectionManager.setMaxTotal(maxConnTotal); + poolingmgr.setMaxTotal(maxConnTotal); } if (maxConnPerRoute > 0) { - connectionManager.setDefaultMaxPerRoute(maxConnPerRoute); + poolingmgr.setDefaultMaxPerRoute(maxConnPerRoute); } - return connectionManager; + return poolingmgr; } catch (final IllegalAccessException e) { throw new RuntimeException(e); } } + + private static String[] split(final String s) { + if (TextUtils.isBlank(s)) { + return null; + } + return s.split(" *, *"); + } + + @SuppressWarnings("unchecked") + private static <T> T getField(final Object target, final String fieldName) throws IllegalAccessException { + return (T) FieldUtils.readDeclaredField(target, fieldName, true); + } } |
From: <rb...@us...> - 2016-07-30 15:27:48
|
Revision: 12889 http://sourceforge.net/p/htmlunit/code/12889 Author: rbri Date: 2016-07-30 15:27:46 +0000 (Sat, 30 Jul 2016) Log Message: ----------- code cleanup Modified Paths: -------------- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java Modified: trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java =================================================================== --- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2016-07-29 19:06:11 UTC (rev 12888) +++ trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2016-07-30 15:27:46 UTC (rev 12889) @@ -131,7 +131,7 @@ private static final String HACKED_COOKIE_POLICY = "mine"; - // have one per thread because this is (re)configured for every call (see configureHttpProcessor) + // have one per thread because this is (re)configured for every call (see configureHttpProcessorBuilder) private ThreadLocal<HttpClientBuilder> httpClientBuilder_ = new ThreadLocal<>(); private final WebClient webClient_; @@ -934,8 +934,6 @@ * but with the ability to configure {@code socketFactory}. */ private static PoolingHttpClientConnectionManager createConnectionManager(final HttpClientBuilder builder) { - final ConnectionSocketFactory socketFactory = new SocksConnectionSocketFactory(); - try { PublicSuffixMatcher publicSuffixMatcher = getField(builder, "publicSuffixMatcher"); if (publicSuffixMatcher == null) { @@ -987,7 +985,7 @@ final PoolingHttpClientConnectionManager poolingmgr = new PoolingHttpClientConnectionManager( RegistryBuilder.<ConnectionSocketFactory>create() - .register("http", socketFactory) + .register("http", new SocksConnectionSocketFactory()) .register("https", sslSocketFactory) .build(), null, |
From: <rb...@us...> - 2016-08-15 18:20:24
|
Revision: 12899 http://sourceforge.net/p/htmlunit/code/12899 Author: rbri Date: 2016-08-15 18:20:21 +0000 (Mon, 15 Aug 2016) Log Message: ----------- use a static WeakHashMap instead of a ThreadLocal because there is no need to access the stored HttpClientBuilder from outside this class Modified Paths: -------------- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java Modified: trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java =================================================================== --- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2016-08-15 17:43:10 UTC (rev 12898) +++ trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2016-08-15 18:20:21 UTC (rev 12899) @@ -137,7 +137,8 @@ private static final String HACKED_COOKIE_POLICY = "mine"; // have one per thread because this is (re)configured for every call (see configureHttpProcessorBuilder) - private ThreadLocal<HttpClientBuilder> httpClientBuilder_ = new ThreadLocal<>(); + // do not use a ThreadLocal because this in only accessed form this class + private static final Map<Thread, HttpClientBuilder> httpClientBuilder_ = new WeakHashMap<>(); private final WebClient webClient_; private String virtualHost_; @@ -206,7 +207,7 @@ // Calling code may catch the StackOverflowError, but due to the leak, the httpClient_ may // come out of connections and throw a ConnectionPoolTimeoutException. // => best solution, discard the HttpClient instance. - httpClientBuilder_.set(null); + httpClientBuilder_.remove(Thread.currentThread()); throw e; } @@ -522,7 +523,7 @@ * @return the initialized HTTP client */ protected HttpClientBuilder getHttpClientBuilder() { - HttpClientBuilder builder = httpClientBuilder_.get(); + HttpClientBuilder builder = httpClientBuilder_.get(Thread.currentThread()); if (builder == null) { builder = createHttpClient(); @@ -534,7 +535,7 @@ builder.setDefaultCookieSpecRegistry(registeryBuilder.build()); builder.setDefaultCookieStore(new HtmlUnitCookieStore(webClient_.getCookieManager())); - httpClientBuilder_.set(builder); + httpClientBuilder_.put(Thread.currentThread(), builder); } return builder; @@ -975,8 +976,9 @@ */ @Override public void close() { - if (httpClientBuilder_.get() != null) { - httpClientBuilder_.set(null); + final Thread current = Thread.currentThread(); + if (httpClientBuilder_.get(current) != null) { + httpClientBuilder_.remove(current); } if (connectionManager_ != null) { connectionManager_.shutdown(); |
From: <rb...@us...> - 2016-08-16 13:30:29
|
Revision: 12917 http://sourceforge.net/p/htmlunit/code/12917 Author: rbri Date: 2016-08-16 13:30:27 +0000 (Tue, 16 Aug 2016) Log Message: ----------- fix my mistake Modified Paths: -------------- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java Modified: trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java =================================================================== --- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2016-08-16 12:49:31 UTC (rev 12916) +++ trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2016-08-16 13:30:27 UTC (rev 12917) @@ -138,7 +138,7 @@ // have one per thread because this is (re)configured for every call (see configureHttpProcessorBuilder) // do not use a ThreadLocal because this in only accessed form this class - private static final Map<Thread, HttpClientBuilder> httpClientBuilder_ = new WeakHashMap<>(); + private final Map<Thread, HttpClientBuilder> httpClientBuilder_ = new WeakHashMap<>(); private final WebClient webClient_; private String virtualHost_; |
From: <rb...@us...> - 2016-11-07 18:01:13
|
Revision: 13063 http://sourceforge.net/p/htmlunit/code/13063 Author: rbri Date: 2016-11-07 18:01:00 +0000 (Mon, 07 Nov 2016) Log Message: ----------- do not hide io exceptions Issue 1832 Modified Paths: -------------- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java Modified: trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java =================================================================== --- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2016-10-28 13:30:46 UTC (rev 13062) +++ trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2016-11-07 18:01:00 UTC (rev 13063) @@ -19,6 +19,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.EOFException; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -718,8 +719,9 @@ * @param is the stream to read * @param maxInMemory the maximumBytes to store in memory, after which save to a local file * @return a wrapper around the downloaded content + * @throws IOException in case of read issues */ - public static DownloadedContent downloadContent(final InputStream is, final int maxInMemory) { + public static DownloadedContent downloadContent(final InputStream is, final int maxInMemory) throws IOException { if (is == null) { return new DownloadedContent.InMemory(new byte[] {}); } @@ -746,9 +748,9 @@ LOG.warn("Connection was closed while reading from stream.", e); return new DownloadedContent.InMemory(bos.toByteArray()); } - catch (final IOException e) { + catch (final EOFException e) { // this might happen with broken gzip content - LOG.warn("Exception while reading from stream.", e); + LOG.warn("EOFException while reading from stream.", e); return new DownloadedContent.InMemory(bos.toByteArray()); } finally { |
From: <rb...@us...> - 2016-11-21 07:20:52
|
Revision: 13080 http://sourceforge.net/p/htmlunit/code/13080 Author: rbri Date: 2016-11-21 07:20:50 +0000 (Mon, 21 Nov 2016) Log Message: ----------- code cleanup Modified Paths: -------------- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java Modified: trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java =================================================================== --- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2016-11-20 16:19:46 UTC (rev 13079) +++ trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2016-11-21 07:20:50 UTC (rev 13080) @@ -708,7 +708,7 @@ protected DownloadedContent downloadResponseBody(final HttpResponse httpResponse) throws IOException { final HttpEntity httpEntity = httpResponse.getEntity(); if (httpEntity == null) { - return new DownloadedContent.InMemory(new byte[] {}); + return new DownloadedContent.InMemory(null); } return downloadContent(httpEntity.getContent(), webClient_.getOptions().getMaxInMemory()); @@ -723,7 +723,7 @@ */ public static DownloadedContent downloadContent(final InputStream is, final int maxInMemory) throws IOException { if (is == null) { - return new DownloadedContent.InMemory(new byte[] {}); + return new DownloadedContent.InMemory(null); } final ByteArrayOutputStream bos = new ByteArrayOutputStream(); |
From: <rb...@us...> - 2017-06-07 18:21:16
|
Revision: 14573 http://sourceforge.net/p/htmlunit/code/14573 Author: rbri Date: 2017-06-07 18:21:14 +0000 (Wed, 07 Jun 2017) Log Message: ----------- code style Modified Paths: -------------- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java Modified: trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java =================================================================== --- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2017-06-07 18:14:44 UTC (rev 14572) +++ trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2017-06-07 18:21:14 UTC (rev 14573) @@ -889,6 +889,7 @@ request.setHeader("Accept-Language", value_); } } + private static final class UpgradeInsecureRequestHeaderHttpRequestInterceptor implements HttpRequestInterceptor { private String value_; |
From: <rb...@us...> - 2017-07-23 18:05:19
|
Revision: 14695 http://sourceforge.net/p/htmlunit/code/14695 Author: rbri Date: 2017-07-23 18:05:17 +0000 (Sun, 23 Jul 2017) Log Message: ----------- revert last fix, it is more complicated Modified Paths: -------------- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java Modified: trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java =================================================================== --- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2017-07-23 17:54:47 UTC (rev 14694) +++ trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2017-07-23 18:05:17 UTC (rev 14695) @@ -97,7 +97,6 @@ import org.apache.http.entity.mime.MultipartEntityBuilder; import org.apache.http.entity.mime.content.InputStreamBody; import org.apache.http.impl.client.BasicAuthCache; -import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.protocol.HttpContext; @@ -192,17 +191,13 @@ HttpResponse httpResponse = null; try { - try (CloseableHttpClient httpClient = builder.build()) { - httpResponse = httpClient.execute(hostConfiguration, httpMethod, httpContext); - } + httpResponse = builder.build().execute(hostConfiguration, httpMethod, httpContext); } catch (final SSLPeerUnverifiedException s) { // Try to use only SSLv3 instead if (webClient_.getOptions().isUseInsecureSSL()) { HtmlUnitSSLConnectionSocketFactory.setUseSSL3Only(httpContext, true); - try (CloseableHttpClient httpClient = builder.build()) { - httpResponse = httpClient.execute(hostConfiguration, httpMethod, httpContext); - } + httpResponse = builder.build().execute(hostConfiguration, httpMethod, httpContext); } else { throw s; |
From: <rb...@us...> - 2018-02-01 19:20:02
|
Revision: 15106 http://sourceforge.net/p/htmlunit/code/15106 Author: rbri Date: 2018-02-01 19:20:00 +0000 (Thu, 01 Feb 2018) Log Message: ----------- try to no longer 'sweep errors under the mat' Issue 1832 Modified Paths: -------------- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java Modified: trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java =================================================================== --- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2018-01-30 19:40:11 UTC (rev 15105) +++ trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2018-02-01 19:20:00 UTC (rev 15106) @@ -715,36 +715,37 @@ if (is == null) { return new DownloadedContent.InMemory(null); } - final ByteArrayOutputStream bos = new ByteArrayOutputStream(); - final byte[] buffer = new byte[1024]; - int nbRead; - try { - while ((nbRead = is.read(buffer)) != -1) { - bos.write(buffer, 0, nbRead); - if (bos.size() > maxInMemory) { - // we have exceeded the max for memory, let's write everything to a temporary file - final File file = File.createTempFile("htmlunit", ".tmp"); - file.deleteOnExit(); - try (FileOutputStream fos = new FileOutputStream(file)) { - bos.writeTo(fos); // what we have already read - IOUtils.copyLarge(is, fos); // what remains from the server response + try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) { + final byte[] buffer = new byte[1024]; + int nbRead; + try { + while ((nbRead = is.read(buffer)) != -1) { + bos.write(buffer, 0, nbRead); + if (bos.size() > maxInMemory) { + // we have exceeded the max for memory, let's write everything to a temporary file + final File file = File.createTempFile("htmlunit", ".tmp"); + file.deleteOnExit(); + try (FileOutputStream fos = new FileOutputStream(file)) { + bos.writeTo(fos); // what we have already read + IOUtils.copyLarge(is, fos); // what remains from the server response + } + return new DownloadedContent.OnFile(file, true); } - return new DownloadedContent.OnFile(file, true); } } - } - catch (final ConnectionClosedException e) { - LOG.warn("Connection was closed while reading from stream.", e); + catch (final ConnectionClosedException e) { + LOG.warn("Connection was closed while reading from stream.", e); + throw e; + } + catch (final EOFException e) { + // this might happen with broken gzip content + LOG.warn("EOFException while reading from stream.", e); + throw e; + } + return new DownloadedContent.InMemory(bos.toByteArray()); } - catch (final EOFException e) { - // this might happen with broken gzip content - LOG.warn("EOFException while reading from stream.", e); - return new DownloadedContent.InMemory(bos.toByteArray()); - } - - return new DownloadedContent.InMemory(bos.toByteArray()); } /** |
From: <rb...@us...> - 2018-02-02 18:47:31
|
Revision: 15111 http://sourceforge.net/p/htmlunit/code/15111 Author: rbri Date: 2018-02-02 18:47:28 +0000 (Fri, 02 Feb 2018) Log Message: ----------- back to a working version, i think we have to add a general error handler to make this customizable Modified Paths: -------------- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java Modified: trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java =================================================================== --- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2018-02-02 18:38:01 UTC (rev 15110) +++ trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2018-02-02 18:47:28 UTC (rev 15111) @@ -736,12 +736,12 @@ } catch (final ConnectionClosedException e) { LOG.warn("Connection was closed while reading from stream.", e); - throw e; + return new DownloadedContent.InMemory(bos.toByteArray()); } catch (final EOFException e) { // this might happen with broken gzip content LOG.warn("EOFException while reading from stream.", e); - throw e; + return new DownloadedContent.InMemory(bos.toByteArray()); } return new DownloadedContent.InMemory(bos.toByteArray()); |
From: <rb...@us...> - 2018-07-18 07:19:27
|
Revision: 15482 http://sourceforge.net/p/htmlunit/code/15482 Author: rbri Date: 2018-07-18 07:19:23 +0000 (Wed, 18 Jul 2018) Log Message: ----------- debug at the server Modified Paths: -------------- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java Modified: trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java =================================================================== --- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2018-07-18 07:14:15 UTC (rev 15481) +++ trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2018-07-18 07:19:23 UTC (rev 15482) @@ -189,7 +189,9 @@ HttpResponse httpResponse = null; try { +System.out.println("#"); httpResponse = builder.build().execute(hostConfiguration, httpMethod, httpContext); +System.out.println("# done"); } catch (final SSLPeerUnverifiedException s) { // Try to use only SSLv3 instead @@ -202,6 +204,7 @@ } } catch (final Error e) { +System.out.println("#" + e); // in case a StackOverflowError occurs while the connection is leased, it won't get released. // Calling code may catch the StackOverflowError, but due to the leak, the httpClient_ may // come out of connections and throw a ConnectionPoolTimeoutException. |
From: <rb...@us...> - 2018-07-18 07:23:21
|
Revision: 15483 http://sourceforge.net/p/htmlunit/code/15483 Author: rbri Date: 2018-07-18 07:23:18 +0000 (Wed, 18 Jul 2018) Log Message: ----------- debug at the server Modified Paths: -------------- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java Modified: trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java =================================================================== --- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2018-07-18 07:19:23 UTC (rev 15482) +++ trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2018-07-18 07:23:18 UTC (rev 15483) @@ -212,6 +212,10 @@ httpClientBuilder_.remove(Thread.currentThread()); throw e; } + catch (final Throwable e) { +e.printStackTrace(); + throw e; + } final DownloadedContent downloadedBody = downloadResponseBody(httpResponse); final long endTime = System.currentTimeMillis(); |
From: <rb...@us...> - 2018-07-18 07:26:49
|
Revision: 15484 http://sourceforge.net/p/htmlunit/code/15484 Author: rbri Date: 2018-07-18 07:26:45 +0000 (Wed, 18 Jul 2018) Log Message: ----------- debug at the server Modified Paths: -------------- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java Modified: trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java =================================================================== --- trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2018-07-18 07:23:18 UTC (rev 15483) +++ trunk/htmlunit/src/main/java/com/gargoylesoftware/htmlunit/HttpWebConnection.java 2018-07-18 07:26:45 UTC (rev 15484) @@ -204,7 +204,7 @@ } } catch (final Error e) { -System.out.println("#" + e); +System.out.println("##" + e); // in case a StackOverflowError occurs while the connection is leased, it won't get released. // Calling code may catch the StackOverflowError, but due to the leak, the httpClient_ may // come out of connections and throw a ConnectionPoolTimeoutException. @@ -213,7 +213,7 @@ throw e; } catch (final Throwable e) { -e.printStackTrace(); +System.out.println("###" + e); throw e; } |