Using the test class below, the line at ///// 1 ///// throws a NoHttpResponseException (see exception trace with logging at the end of the question). Executing any of the other lines ///// 2-to-8 ///// prints out the headers fine. Is this a bug in HttpClient 4.3.x or am I doing something wrong (I have tested this with both 4.3.1 and 4.3.2)?
import java.util.Arrays;
import org.apache.http.client.methods.*;
import org.apache.http.impl.client.*;
import org.apache.http.params.*;
public class PrintHeaders {
private static final String MONEY_SMART_URL = "https://www.moneysmart.gov.au/";
private static final String TGA_URL = "https://www.ebs.tga.gov.au/ebs/picmi/picmirepository.nsf/PICMI?OpenForm&t=&k=P";
private static final String DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:26.0) Gecko/20100101 Firefox/26.0";
public static void main(String[] args) throws Exception {
printHeadersByHead_4_3(TGA_URL); ///// 1 /////
// printHeadersByHead_4_2(TGA_URL); ///// 2 /////
// printHeadersByGet_4_3(TGA_URL); ///// 3 /////
// printHeadersByGet_4_2(TGA_URL); ///// 4 /////
// printHeadersByHead_4_3(MONEY_SMART_URL); ///// 5 /////
// printHeadersByHead_4_2(MONEY_SMART_URL); ///// 6 /////
// printHeadersByGet_4_3(MONEY_SMART_URL); ///// 7 /////
// printHeadersByGet_4_2(MONEY_SMART_URL); ///// 8 /////
}
public static void printHeadersByHead_4_3(String docURL) {
printHeaders(new HttpHead(docURL), buildHttpClient_4_3());
}
public static void printHeadersByHead_4_2(String docURL) {
printHeaders(new HttpHead(docURL), buildHttpClient_4_2());
}
public static void printHeadersByGet_4_3(String docURL) {
printHeaders(new HttpGet(docURL), buildHttpClient_4_3());
}
public static void printHeadersByGet_4_2(String docURL) {
printHeaders(new HttpGet(docURL), buildHttpClient_4_2());
}
public static void printHeaders(HttpRequestBase req, CloseableHttpClient client) {
CloseableHttpResponse response = null;
try {
try {
response = client.execute(req);
System.out.println(Arrays.asList(response.getAllHeaders()));
} finally {
if (response != null) response.close();
if (client != null) client.close();
}
} catch (Exception e) {
e.printStackTrace();
}
}
public static CloseableHttpClient buildHttpClient_4_3() {
return HttpClients.custom().setUserAgent(DEFAULT_USER_AGENT).build();
}
@SuppressWarnings("deprecation")
public static CloseableHttpClient buildHttpClient_4_2() {
AbstractHttpClient httpClient = new DefaultHttpClient();
httpClient.getParams().setParameter(CoreProtocolPNames.USER_AGENT, DEFAULT_USER_AGENT);
return httpClient;
}
}
EXCEPTION TRACE when running line at ///// 1 /////:
2014/02/17 09:36:43:857 EST [DEBUG] RequestAddCookies - CookieSpec selected: best-match
2014/02/17 09:36:43:871 EST [DEBUG] RequestAuthCache - Auth cache not set in the context
2014/02/17 09:36:43:873 EST [DEBUG] PoolingHttpClientConnectionManager - Connection request: [route: {s}->https://www.ebs.tga.gov.au:443][total kept alive: 0; route allocated: 0 of 2; total allocated: 0 of 20]
2014/02/17 09:36:43:890 EST [DEBUG] PoolingHttpClientConnectionManager - Connection leased: [id: 0][route: {s}->https://www.ebs.tga.gov.au:443][total kept alive: 0; route allocated: 1 of 2; total allocated: 1 of 20]
2014/02/17 09:36:43:902 EST [DEBUG] MainClientExec - Opening connection {s}->https://www.ebs.tga.gov.au:443
2014/02/17 09:36:44:046 EST [DEBUG] HttpClientConnectionManager - Connecting to www.ebs.tga.gov.au/161.146.233.4:443
2014/02/17 09:36:45:038 EST [DEBUG] MainClientExec - Executing request HEAD /ebs/picmi/picmirepository.nsf/PICMI?OpenForm&t=&k=P HTTP/1.1
2014/02/17 09:36:45:038 EST [DEBUG] MainClientExec - Target auth state: UNCHALLENGED
2014/02/17 09:36:45:039 EST [DEBUG] MainClientExec - Proxy auth state: UNCHALLENGED
2014/02/17 09:36:45:041 EST [DEBUG] headers - http-outgoing-0 >> HEAD /ebs/picmi/picmirepository.nsf/PICMI?OpenForm&t=&k=P HTTP/1.1
2014/02/17 09:36:45:041 EST [DEBUG] headers - http-outgoing-0 >> Host: www.ebs.tga.gov.au
2014/02/17 09:36:45:041 EST [DEBUG] headers - http-outgoing-0 >> Connection: Keep-Alive
2014/02/17 09:36:45:041 EST [DEBUG] headers - http-outgoing-0 >> User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64; rv:26.0) Gecko/20100101 Firefox/26.0
2014/02/17 09:36:45:042 EST [DEBUG] headers - http-outgoing-0 >> Accept-Encoding: gzip,deflate
2014/02/17 09:37:00:222 EST [DEBUG] DefaultManagedHttpClientConnection - http-outgoing-0: Close connection
2014/02/17 09:37:00:222 EST [DEBUG] DefaultManagedHttpClientConnection - http-outgoing-0: Shutdown connection
2014/02/17 09:37:00:222 EST [DEBUG] MainClientExec - Connection discarded
2014/02/17 09:37:00:222 EST [DEBUG] DefaultManagedHttpClientConnection - http-outgoing-0: Close connection
2014/02/17 09:37:00:223 EST [DEBUG] PoolingHttpClientConnectionManager - Connection released: [id: 0][route: {s}->https://www.ebs.tga.gov.au:443][total kept alive: 0; route allocated: 0 of 2; total allocated: 0 of 20]
2014/02/17 09:37:00:224 EST [INFO] RetryExec - I/O exception (org.apache.http.NoHttpResponseException) caught when processing request: The target server failed to respond
2014/02/17 09:37:00:224 EST [DEBUG] RetryExec - The target server failed to respond <org.apache.http.NoHttpResponseException: The target server failed to respond>org.apache.http.NoHttpResponseException: The target server failed to respond
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:161)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.http.impl.conn.CPoolProxy.invoke(CPoolProxy.java:138)
at com.sun.proxy.$Proxy0.receiveResponseHeader(Unknown Source)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:253)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:194)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:85)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:108)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:186)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
at PerformHttpHead.printHeaders(PrintHeaders.java:43)
at PerformHttpHead.printHeadersByHead_4_3(PrintHeaders.java:24)
at PerformHttpHead.main(PrintHeaders.java:13)
The culprit turned out to be automatic content decompression, which is off by default in 4.2 and on in 4.3. Presence of Accept-Encoding: gzip,deflate in HEAD requests seems to confuse Lotus-Domino web server.
[DEBUG] RequestAddCookies - CookieSpec selected: best-match
[DEBUG] RequestAuthCache - Auth cache not set in the context
[DEBUG] PoolingHttpClientConnectionManager - Connection request: [route: {s}->https://www.ebs.tga.gov.au:443][total kept alive: 0; route allocated: 0 of 100; total allocated: 0 of 200]
[DEBUG] PoolingHttpClientConnectionManager - Connection leased: [id: 0][route: {s}->https://www.ebs.tga.gov.au:443][total kept alive: 0; route allocated: 1 of 100; total allocated: 1 of 200]
[DEBUG] MainClientExec - Opening connection {s}->https://www.ebs.tga.gov.au:443
[DEBUG] HttpClientConnectionOperator - Connecting to www.ebs.tga.gov.au/161.146.233.4:443
[DEBUG] HttpClientConnectionOperator - Connection established 192.168.42.63:40880<->161.146.233.4:443
[DEBUG] MainClientExec - Executing request HEAD / HTTP/1.1
[DEBUG] MainClientExec - Target auth state: UNCHALLENGED
[DEBUG] MainClientExec - Proxy auth state: UNCHALLENGED
[DEBUG] headers - http-outgoing-0 >> HEAD / HTTP/1.1
[DEBUG] headers - http-outgoing-0 >> Host: www.ebs.tga.gov.au
[DEBUG] headers - http-outgoing-0 >> Connection: Keep-Alive
[DEBUG] headers - http-outgoing-0 >> User-Agent: Apache-HttpClient/4.3.3-SNAPSHOT (java 1.5)
[DEBUG] headers - http-outgoing-0 << HTTP/1.1 200 OK
[DEBUG] headers - http-outgoing-0 << Server: Lotus-Domino
[DEBUG] headers - http-outgoing-0 << Date: Tue, 18 Feb 2014 15:25:32 GMT
[DEBUG] headers - http-outgoing-0 << Content-Type: text/html
[DEBUG] headers - http-outgoing-0 << Content-Length: 15765
[DEBUG] headers - http-outgoing-0 << Expires: Thu, 01 Jan 1970 23:59:59 GMT
[DEBUG] headers - http-outgoing-0 << Cache-control : no-cache, no-store, private Pragma: Value: no-cache
[DEBUG] MainClientExec - Connection can be kept alive indefinitely
[DEBUG] PoolingHttpClientConnectionManager - Connection [id: 0][route: {s}->https://www.ebs.tga.gov.au:443] can be kept alive indefinitely
[DEBUG] PoolingHttpClientConnectionManager - Connection released: [id: 0][route: {s}->https://www.ebs.tga.gov.au:443][total kept alive: 1; route allocated: 1 of 100; total allocated: 1 of 200]