    1   /*
    2    * $Header: /home/jerenkrantz/tmp/commons/commons-convert/cvs/home/cvs/jakarta-commons//httpclient/src/java/org/apache/commons/httpclient/HttpConnection.java,v 1.107 2005/01/14 21:30:59 olegk Exp $
    3    * $Revision: 5562 $
    4    * $Date: 2007-11-16 00:53:10 +0000 (Fri, 16 Nov 2007) $
    5    *
    6    * ====================================================================
    7    *
    8    *  Copyright 1999-2004 The Apache Software Foundation
    9    *
    10   *  Licensed under the Apache License, Version 2.0 (the "License");
    11   *  you may not use this file except in compliance with the License.
    12   *  You may obtain a copy of the License at
    13   *
    14   *      http://www.apache.org/licenses/LICENSE-2.0
    15   *
    16   *  Unless required by applicable law or agreed to in writing, software
    17   *  distributed under the License is distributed on an "AS IS" BASIS,
    18   *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    19   *  See the License for the specific language governing permissions and
    20   *  limitations under the License.
    21   * ====================================================================
    22   *
    23   * This software consists of voluntary contributions made by many
    24   * individuals on behalf of the Apache Software Foundation.  For more
    25   * information on the Apache Software Foundation, please see
    26   * <http://www.apache.org/>.
    27   *
    28   */
    29
    30  package org.apache.commons.httpclient;
    31
    32  import java.io.BufferedInputStream;
    33  import java.io.BufferedOutputStream;
    34  import java.io.IOException;
    35  import java.io.InputStream;
    36  import java.io.InterruptedIOException;
    37  import java.io.OutputStream;
    38  import java.lang.reflect.Method;
    39  import java.net.InetAddress;
    40  import java.net.Socket;
    41  import java.net.SocketException;
    42
    43  import org.apache.commons.httpclient.params.HttpConnectionParams;
    44  import org.apache.commons.httpclient.protocol.Protocol;
    45  import org.apache.commons.httpclient.protocol.ProtocolSocketFactory;
    46  import org.apache.commons.httpclient.protocol.SecureProtocolSocketFactory;
    47  import org.apache.commons.httpclient.util.EncodingUtil;
    48  import org.apache.commons.httpclient.util.ExceptionUtil;
    49  import org.apache.commons.logging.Log;
    50  import org.apache.commons.logging.LogFactory;
    51
    52  import org.archive.util.HttpRecorder; // <- // IA/HERITRIX import
    53
    54  /***
    55   * An abstraction of an HTTP {@link InputStream} and {@link OutputStream}
    56   * pair, together with the relevant attributes.
    57   * <p>
    58   * The following options are set on the socket before getting the input/output
    59   * streams in the {@link #open()} method:
    60   * <table border=1><tr>
    61   *    <th>Socket Method
    62   *    <th>Sockets Option
    63   *    <th>Configuration
    64   * </tr><tr>
    65   *    <td>{@link java.net.Socket#setTcpNoDelay(boolean)}
    66   *    <td>SO_NODELAY
    67   *    <td>{@link HttpConnectionParams#setTcpNoDelay(boolean)}
    68   * </tr><tr>
    69   *    <td>{@link java.net.Socket#setSoTimeout(int)}
    70   *    <td>SO_TIMEOUT
    71   *    <td>{@link HttpConnectionParams#setSoTimeout(int)}
    72   * </tr><tr>
    73   *    <td>{@link java.net.Socket#setSendBufferSize(int)}
    74   *    <td>SO_SNDBUF
    75   *    <td>{@link HttpConnectionParams#setSendBufferSize(int)}
    76   * </tr><tr>
    77   *    <td>{@link java.net.Socket#setReceiveBufferSize(int)}
    78   *    <td>SO_RCVBUF
    79   *    <td>{@link HttpConnectionParams#setReceiveBufferSize(int)}
    80   * </tr></table>
    81   *
    82   * @author Rod Waldhoff
    83   * @author Sean C. Sullivan
    84   * @author Ortwin Glueck
    85   * @author <a href="mailto:jsdever@apache.org">Jeff Dever</a>
    86   * @author <a href="mailto:mbowler@GargoyleSoftware.com">Mike Bowler</a>
    87   * @author <a href="mailto:oleg@ural.ru">Oleg Kalnichevski</a>
    88   * @author Michael Becke
    89   * @author Eric E Johnson
    90   * @author Laura Werner
    91   *
    92   * @version   $Revision: 5562 $ $Date: 2007-11-16 00:53:10 +0000 (Fri, 16 Nov 2007) $
    93   */
    94  public class HttpConnection {
    95
    96      // ----------------------------------------------------------- Constructors
    97
    98      /***
    99       * Creates a new HTTP connection for the given host and port.
    100      *
    101      * @param host the host to connect to
    102      * @param port the port to connect to
    103      */
    104     public HttpConnection(String host, int port) {
    105         this(null, -1, host, null, port, Protocol.getProtocol("http"));
    106     }
    107
    108     /***
    109      * Creates a new HTTP connection for the given host and port
    110      * using the given protocol.
    111      *
    112      * @param host the host to connect to
    113      * @param port the port to connect to
    114      * @param protocol the protocol to use
    115      */
    116     public HttpConnection(String host, int port, Protocol protocol) {
    117         this(null, -1, host, null, port, protocol);
    118     }
    119
    120     /***
    121      * Creates a new HTTP connection for the given host with the virtual
    122      * alias and port using given protocol.
    123      *
    124      * @param host the host to connect to
    125      * @param virtualHost the virtual host requests will be sent to
    126      * @param port the port to connect to
    127      * @param protocol the protocol to use
    128      */
    129     public HttpConnection(String host, String virtualHost, int port, Protocol protocol) {
    130         this(null, -1, host, virtualHost, port, protocol);
    131     }
    132
    133     /***
    134      * Creates a new HTTP connection for the given host and port via the
    135      * given proxy host and port using the default protocol.
    136      *
    137      * @param proxyHost the host to proxy via
    138      * @param proxyPort the port to proxy via
    139      * @param host the host to connect to
    140      * @param port the port to connect to
    141      */
    142     public HttpConnection(
    143         String proxyHost,
    144         int proxyPort,
    145         String host,
    146         int port) {
    147         this(proxyHost, proxyPort, host, null, port, Protocol.getProtocol("http"));
    148     }
    149
    150     /***
    151      * Creates a new HTTP connection for the given host configuration.
    152      *
    153      * @param hostConfiguration the host/proxy/protocol to use
    154      */
    155     public HttpConnection(HostConfiguration hostConfiguration) {
    156         this(hostConfiguration.getProxyHost(),
    157              hostConfiguration.getProxyPort(),
    158              hostConfiguration.getHost(),
    159              hostConfiguration.getPort(),
    160              hostConfiguration.getProtocol());
    161         this.localAddress = hostConfiguration.getLocalAddress();
    162     }
    163
    164     /***
    165      * Creates a new HTTP connection for the given host with the virtual
    166      * alias and port via the given proxy host and port using the given
    167      * protocol.
    168      *
    169      * @param proxyHost the host to proxy via
    170      * @param proxyPort the port to proxy via
    171      * @param host the host to connect to. Parameter value must be non-null.
    172      * @param virtualHost No longer applicable.
    173      * @param port the port to connect to
    174      * @param protocol The protocol to use. Parameter value must be non-null.
    175      *
    176      * @deprecated use #HttpConnection(String, int, String, int, Protocol)
    177      */
    178     public HttpConnection(
    179         String proxyHost,
    180         int proxyPort,
    181         String host,
    182         String virtualHost,
    183         int port,
    184         Protocol protocol) {
    185         this(proxyHost, proxyPort, host, port, protocol);
    186     }
    187
    188     /***
    189      * Creates a new HTTP connection for the given host with the virtual
    190      * alias and port via the given proxy host and port using the given
    191      * protocol.
    192      *
    193      * @param proxyHost the host to proxy via
    194      * @param proxyPort the port to proxy via
    195      * @param host the host to connect to. Parameter value must be non-null.
    196      * @param port the port to connect to
    197      * @param protocol The protocol to use. Parameter value must be non-null.
    198      */
    199     public HttpConnection(
    200         String proxyHost,
    201         int proxyPort,
    202         String host,
    203         int port,
    204         Protocol protocol) {
    205
    206         if (host == null) {
    207             throw new IllegalArgumentException("host parameter is null");
    208         }
    209         if (protocol == null) {
    210             throw new IllegalArgumentException("protocol is null");
    211         }
    212
    213         proxyHostName = proxyHost;
    214         proxyPortNumber = proxyPort;
    215         hostName = host;
    216         portNumber = protocol.resolvePort(port);
    217         protocolInUse = protocol;
    218     }
    219
    220     // ------------------------------------------ Attribute Setters and Getters
    221
    222     /***
    223      * Returns the connection socket.
    224      *
    225      * @return the socket.
    226      *
    227      * @since 3.0
    228      */
    229     protected Socket getSocket() {
    230         return this.socket;
    231     }
    232
    233     /***
    234      * Returns the host.
    235      *
    236      * @return the host.
    237      */
    238     public String getHost() {
    239         return hostName;
    240     }
    241
    242     /***
    243      * Sets the host to connect to.
    244      *
    245      * @param host the host to connect to. Parameter value must be non-null.
    246      * @throws IllegalStateException if the connection is already open
    247      */
    248     public void setHost(String host) throws IllegalStateException {
    249         if (host == null) {
    250             throw new IllegalArgumentException("host parameter is null");
    251         }
    252         assertNotOpen();
    253         hostName = host;
    254     }
    255
    256     /***
    257      * Returns the target virtual host.
    258      *
    259      * @return the virtual host.
    260      *
    261      * @deprecated no longer applicable
    262      */
    263
    264     public String getVirtualHost() {
    265         return this.hostName;
    266     }
    267
    268     /***
    269      * Sets the virtual host to target.
    270      *
    271      * @param host the virtual host name that should be used instead of
    272      *        physical host name when sending HTTP requests. Virtual host
    273      *        name can be set to <tt> null</tt> if virtual host name is not
    274      *        to be used
    275      *
    276      * @throws IllegalStateException if the connection is already open
    277      *
    278      * @deprecated no longer applicable
    279      */
    280
    281     public void setVirtualHost(String host) throws IllegalStateException {
    282         assertNotOpen();
    283     }
    284
    285     /***
    286      * Returns the port of the host.
    287      *
    288      * If the port is -1 (or less than 0) the default port for
    289      * the current protocol is returned.
    290      *
    291      * @return the port.
    292      */
    293     public int getPort() {
    294         if (portNumber < 0) {
    295             return isSecure() ? 443 : 80;
    296         } else {
    297             return portNumber;
    298         }
    299     }
    300
    301     /***
    302      * Sets the port to connect to.
    303      *
    304      * @param port the port to connect to
    305      *
    306      * @throws IllegalStateException if the connection is already open
    307      */
    308     public void setPort(int port) throws IllegalStateException {
    309         assertNotOpen();
    310         portNumber = port;
    311     }
    312
    313     /***
    314      * Returns the proxy host.
    315      *
    316      * @return the proxy host.
    317      */
    318     public String getProxyHost() {
    319         return proxyHostName;
    320     }
    321
    322     /***
    323      * Sets the host to proxy through.
    324      *
    325      * @param host the host to proxy through.
    326      *
    327      * @throws IllegalStateException if the connection is already open
    328      */
    329     public void setProxyHost(String host) throws IllegalStateException {
    330         assertNotOpen();
    331         proxyHostName = host;
    332     }
    333
    334     /***
    335      * Returns the port of the proxy host.
    336      *
    337      * @return the proxy port.
    338      */
    339     public int getProxyPort() {
    340         return proxyPortNumber;
    341     }
    342
    343     /***
    344      * Sets the port of the host to proxy through.
    345      *
    346      * @param port the port of the host to proxy through.
    347      *
    348      * @throws IllegalStateException if the connection is already open
    349      */
    350     public void setProxyPort(int port) throws IllegalStateException {
    351         assertNotOpen();
    352         proxyPortNumber = port;
    353     }
    354
    355     /***
    356      * Returns <tt>true</tt> if the connection is established over
    357      * a secure protocol.
    358      *
    359      * @return <tt>true</tt> if connected over a secure protocol.
    360      */
    361     public boolean isSecure() {
    362         return protocolInUse.isSecure();
    363     }
    364
    365     /***
    366      * Returns the protocol used to establish the connection.
    367      * @return The protocol
    368      */
    369     public Protocol getProtocol() {
    370         return protocolInUse;
    371     }
    372
    373     /***
    374      * Sets the protocol used to establish the connection
    375      *
    376      * @param protocol The protocol to use.
    377      *
    378      * @throws IllegalStateException if the connection is already open
    379      */
    380     public void setProtocol(Protocol protocol) {
    381         assertNotOpen();
    382
    383         if (protocol == null) {
    384             throw new IllegalArgumentException("protocol is null");
    385         }
    386
    387         protocolInUse = protocol;
    388
    389     }
    390
    391     /***
    392      * Return the local address used when creating the connection.
    393      * If <tt>null</tt>, the default address is used.
    394      *
    395      * @return InetAddress the local address to be used when creating Sockets
    396      */
    397     public InetAddress getLocalAddress() {
    398         return this.localAddress;
    399     }
    400
    401     /***
    402      * Set the local address used when creating the connection.
    403      * If unset or <tt>null</tt>, the default address is used.
    404      *
    405      * @param localAddress the local address to use
    406      */
    407     public void setLocalAddress(InetAddress localAddress) {
    408         assertNotOpen();
    409         this.localAddress = localAddress;
    410     }
    411
    412     /***
    413      * Tests if the connection is open.
    414      *
    415      * @return <code>true</code> if the connection is open
    416      */
    417     public boolean isOpen() {
    418         return isOpen;
    419     }
    420
    421     /***
    422      * Closes the connection if stale.
    423      *
    424      * @return <code>true</code> if the connection was stale and therefore closed,
    425      * <code>false</code> otherwise.
    426      *
    427      * @see #isStale()
    428      *
    429      * @since 3.0
    430      */
    431     public boolean closeIfStale() throws IOException {
    432         if (isOpen && isStale()) {
    433             LOG.debug("Connection is stale, closing...");
    434             close();
    435             return true;
    436         }
    437         return false;
    438     }
    439
    440     /***
    441      * Tests if stale checking is enabled.
    442      *
    443      * @return <code>true</code> if enabled
    444      *
    445      * @see #isStale()
    446      *
    447      * @deprecated Use {@link HttpConnectionParams#isStaleCheckingEnabled()},
    448      * {@link HttpConnection#getParams()}.
    449      */
    450     public boolean isStaleCheckingEnabled() {
    451         return this.params.isStaleCheckingEnabled();
    452     }
    453
    454     /***
    455      * Sets whether or not isStale() will be called when testing if this connection is open.
    456      *
    457      * <p>Setting this flag to <code>false</code> will increase performance when reusing
    458      * connections, but it will also make them less reliable.  Stale checking ensures that
    459      * connections are viable before they are used.  When set to <code>false</code> some
    460      * method executions will result in IOExceptions and they will have to be retried.</p>
    461      *
    462      * @param staleCheckEnabled <code>true</code> to enable isStale()
    463      *
    464      * @see #isStale()
    465      * @see #isOpen()
    466      *
    467      * @deprecated Use {@link HttpConnectionParams#setStaleCheckingEnabled(boolean)},
    468      * {@link HttpConnection#getParams()}.
    469      */
    470     public void setStaleCheckingEnabled(boolean staleCheckEnabled) {
    471         this.params.setStaleCheckingEnabled(staleCheckEnabled);
    472     }
    473
    474     /***
    475      * Determines whether this connection is "stale", which is to say that either
    476      * it is no longer open, or an attempt to read the connection would fail.
    477      *
    478      * <p>Unfortunately, due to the limitations of the JREs prior to 1.4, it is
    479      * not possible to test a connection to see if both the read and write channels
    480      * are open - except by reading and writing.  This leads to a difficulty when
    481      * some connections leave the "write" channel open, but close the read channel
    482      * and ignore the request.  This function attempts to ameliorate that
    483      * problem by doing a test read, assuming that the caller will be doing a
    484      * write followed by a read, rather than the other way around.
    485      * </p>
    486      *
    487      * <p>To avoid side-effects, the underlying connection is wrapped by a
    488      * {@link BufferedInputStream}, so although data might be read, what is visible
    489      * to clients of the connection will not change with this call.</p.
    490      *
    491      * @throws IOException if the stale connection test is interrupted.
    492      *
    493      * @return <tt>true</tt> if the connection is already closed, or a read would
    494      * fail.
    495      */
    496     protected boolean isStale() throws IOException {
    497         boolean isStale = true;
    498         if (isOpen) {
    499             // the connection is open, but now we have to see if we can read it
    500             // assume the connection is not stale.
    501             isStale = false;
    502             try {
    503                 if (inputStream.available() <= 0) {
    504                     try {
    505                         socket.setSoTimeout(1);
    506                         inputStream.mark(1);
    507                         int byteRead = inputStream.read();
    508                         if (byteRead == -1) {
    509                             // again - if the socket is reporting all data read,
    510                             // probably stale
    511                             isStale = true;
    512                         } else {
    513                             inputStream.reset();
    514                         }
    515                     } finally {
    516                         socket.setSoTimeout(this.params.getSoTimeout());
    517                     }
    518                 }
    519             } catch (InterruptedIOException e) {
    520                 if (!ExceptionUtil.isSocketTimeoutException(e)) {
    521                     throw e;
    522                 }
    523                 // aha - the connection is NOT stale - continue on!
    524             } catch (IOException e) {
    525                 // oops - the connection is stale, the read or soTimeout failed.
    526                 LOG.debug(
    527                     "An error occurred while reading from the socket, is appears to be stale",
    528                     e
    529                 );
    530                 isStale = true;
    531             }
    532         }
    533
    534         return isStale;
    535     }
    536
    537     /***
    538      * Returns <tt>true</tt> if the connection is established via a proxy,
    539      * <tt>false</tt> otherwise.
    540      *
    541      * @return <tt>true</tt> if a proxy is used to establish the connection,
    542      * <tt>false</tt> otherwise.
    543      */
    544     public boolean isProxied() {
    545         return (!(null == proxyHostName || 0 >= proxyPortNumber));
    546     }
    547
    548     /***
    549      * Set the state to keep track of the last response for the last request.
    550      *
    551      * <p>The connection managers use this to ensure that previous requests are
    552      * properly closed before a new request is attempted.  That way, a GET
    553      * request need not be read in its entirety before a new request is issued.
    554      * Instead, this stream can be closed as appropriate.</p>
    555      *
    556      * @param inStream  The stream associated with an HttpMethod.
    557      */
    558     public void setLastResponseInputStream(InputStream inStream) {
    559         lastResponseInputStream = inStream;
    560     }
    561
    562     /***
    563      * Returns the stream used to read the last response's body.
    564      *
    565      * <p>Clients will generally not need to call this function unless
    566      * using HttpConnection directly, instead of calling {@link HttpClient#executeMethod}.
    567      * For those clients, call this function, and if it returns a non-null stream,
    568      * close the stream before attempting to execute a method.  Note that
    569      * calling "close" on the stream returned by this function <i>may</i> close
    570      * the connection if the previous response contained a "Connection: close" header. </p>
    571      *
    572      * @return An {@link InputStream} corresponding to the body of the last
    573      *  response.
    574      */
    575     public InputStream getLastResponseInputStream() {
    576         return lastResponseInputStream;
    577     }
    578
    579     // --------------------------------------------------- Other Public Methods
    580
    581     /***
    582      * Returns {@link HttpConnectionParams HTTP protocol parameters} associated with this method.
    583      *
    584      * @return HTTP parameters.
    585      *
    586      * @since 3.0
    587      */
    588     public HttpConnectionParams getParams() {
    589         return this.params;
    590     }
    591
    592     /***
    593      * Assigns {@link HttpConnectionParams HTTP protocol parameters} for this method.
    594      *
    595      * @since 3.0
    596      *
    597      * @see HttpConnectionParams
    598      */
    599     public void setParams(final HttpConnectionParams params) {
    600         if (params == null) {
    601             throw new IllegalArgumentException("Parameters may not be null");
    602         }
    603         this.params = params;
    604     }
    605
    606     /***
    607      * Set the {@link Socket}'s timeout, via {@link Socket#setSoTimeout}.  If the
    608      * connection is already open, the SO_TIMEOUT is changed.  If no connection
    609      * is open, then subsequent connections will use the timeout value.
    610      * <p>
    611      * Note: This is not a connection timeout but a timeout on network traffic!
    612      *
    613      * @param timeout the timeout value
    614      * @throws SocketException - if there is an error in the underlying
    615      * protocol, such as a TCP error.
    616      *
    617      * @deprecated Use {@link HttpConnectionParams#setSoTimeout(int)},
    618      * {@link HttpConnection#getParams()}.
    619      */
    620     public void setSoTimeout(int timeout)
    621         throws SocketException, IllegalStateException {
    622         this.params.setSoTimeout(timeout);
    623         if (this.socket != null) {
    624             this.socket.setSoTimeout(timeout);
    625         }
    626     }
    627
    628     /***
    629      * Sets <code>SO_TIMEOUT</code> value directly on the underlying {@link Socket socket}.
    630      * This method does not change the default read timeout value set via
    631      * {@link HttpConnectionParams}.
    632      *
    633      * @param timeout the timeout value
    634      * @throws SocketException - if there is an error in the underlying
    635      * protocol, such as a TCP error.
    636      * @throws IllegalStateException if not connected
    637      *
    638      * @since 3.0
    639      */
    640     public void setSocketTimeout(int timeout)
    641         throws SocketException, IllegalStateException {
    642         assertOpen();
    643         if (this.socket != null) {
    644             this.socket.setSoTimeout(timeout);
    645         }
    646     }
    647
    648     /***
    649      * Returns the {@link Socket}'s timeout, via {@link Socket#getSoTimeout}, if the
    650      * connection is already open. If no connection is open, return the value subsequent
    651      * connection will use.
    652      * <p>
    653      * Note: This is not a connection timeout but a timeout on network traffic!
    654      *
    655      * @return the timeout value
    656      *
    657      * @deprecated Use {@link HttpConnectionParams#getSoTimeout()},
    658      * {@link HttpConnection#getParams()}.
    659      */
    660     public int getSoTimeout() throws SocketException {
    661         return this.params.getSoTimeout();
    662     }
    663
    664     /***
    665      * Sets the connection timeout. This is the maximum time that may be spent
    666      * until a connection is established. The connection will fail after this
    667      * amount of time.
    668      * @param timeout The timeout in milliseconds. 0 means timeout is not used.
    669      *
    670      * @deprecated Use {@link HttpConnectionParams#setConnectionTimeout(int)},
    671      * {@link HttpConnection#getParams()}.
    672      */
    673     public void setConnectionTimeout(int timeout) {
    674         this.params.setConnectionTimeout(timeout);
    675     }
    676
    677     /***
    678      * Establishes a connection to the specified host and port
    679      * (via a proxy if specified).
    680      * The underlying socket is created from the {@link ProtocolSocketFactory}.
    681      *
    682      * @throws IOException if an attempt to establish the connection results in an
    683      *   I/O error.
    684      */
    685     public void open() throws IOException {
    686         LOG.trace("enter HttpConnection.open()");
    687
    688         final String host = (proxyHostName == null) ? hostName : proxyHostName;
    689         final int port = (proxyHostName == null) ? portNumber : proxyPortNumber;
    690         assertNotOpen();
    691
    692         if (LOG.isDebugEnabled()) {
    693             LOG.debug("Open connection to " + host + ":" + port);
    694         }
    695
    696         try {
    697             if (this.socket == null) {
    698                 usingSecureSocket = isSecure() && !isProxied();
    699                 // use the protocol's socket factory unless this is a secure
    700                 // proxied connection
    701                 ProtocolSocketFactory socketFactory = null;
    702                 if (isSecure() && isProxied()) {
    703                     Protocol defaultprotocol = Protocol.getProtocol("http");
    704                     socketFactory = defaultprotocol.getSocketFactory();
    705                 } else {
    706                     socketFactory = this.protocolInUse.getSocketFactory();
    707                 }
    708                 this.socket = socketFactory.createSocket(
    709                             host, port,
    710                             localAddress, 0,
    711                             this.params);
    712             }
    713
    714             /*
    715             "Nagling has been broadly implemented across networks,
    716             including the Internet, and is generally performed by default
    717             - although it is sometimes considered to be undesirable in
    718             highly interactive environments, such as some client/server
    719             situations. In such cases, nagling may be turned off through
    720             use of the TCP_NODELAY sockets option." */
    721
    722             socket.setTcpNoDelay(this.params.getTcpNoDelay());
    723             socket.setSoTimeout(this.params.getSoTimeout());
    724
    725             int linger = this.params.getLinger();
    726             if (linger >= 0) {
    727                 socket.setSoLinger(linger > 0, linger);
    728             }
    729
    730             int sndBufSize = this.params.getSendBufferSize();
    731             if (sndBufSize >= 0) {
    732                 socket.setSendBufferSize(sndBufSize);
    733             }
    734             int rcvBufSize = this.params.getReceiveBufferSize();
    735             if (rcvBufSize >= 0) {
    736                 socket.setReceiveBufferSize(rcvBufSize);
    737             }
    738             int outbuffersize = socket.getSendBufferSize();
    739             if ((outbuffersize > 2048) || (outbuffersize <= 0)) {
    740                 outbuffersize = 2048;
    741             }
    742             int inbuffersize = socket.getReceiveBufferSize();
    743             if ((inbuffersize > 2048) || (inbuffersize <= 0)) {
    744                 inbuffersize = 2048;
    745             }
    746
    747             // START IA/HERITRIX change
    748             HttpRecorder httpRecorder = HttpRecorder.getHttpRecorder();
    749             if (httpRecorder == null || (isSecure() && isProxied())) {
    750                 // no recorder, OR defer recording for pre-tunnel leg
    751                 inputStream = new BufferedInputStream(
    752                     socket.getInputStream(), inbuffersize);
    753                 outputStream = new BufferedOutputStream(
    754                     socket.getOutputStream(), outbuffersize);
    755             } else {
    756                 inputStream = httpRecorder.inputWrap((InputStream)
    757                         (new BufferedInputStream(socket.getInputStream(),
    758                         inbuffersize)));
    759                 outputStream = httpRecorder.outputWrap((OutputStream)
    760                         (new BufferedOutputStream(socket.getOutputStream(),
    761                         outbuffersize)));
    762             }
    763             // END IA/HERITRIX change
    764
    765             isOpen = true;
    766         } catch (IOException e) {
    767             // Connection wasn't opened properly
    768             // so close everything out
    769             closeSocketAndStreams();
    770             throw e;
    771         }
    772     }
    773
    774     /***
    775      * Instructs the proxy to establish a secure tunnel to the host. The socket will
    776      * be switched to the secure socket. Subsequent communication is done via the secure
    777      * socket. The method can only be called once on a proxied secure connection.
    778      *
    779      * @throws IllegalStateException if connection is not secure and proxied or
    780      * if the socket is already secure.
    781      * @throws IOException if an attempt to establish the secure tunnel results in an
    782      *   I/O error.
    783      */
    784     public void tunnelCreated() throws IllegalStateException, IOException {
    785         LOG.trace("enter HttpConnection.tunnelCreated()");
    786
    787         if (!isSecure() || !isProxied()) {
    788             throw new IllegalStateException(
    789                 "Connection must be secure "
    790                     + "and proxied to use this feature");
    791         }
    792
    793         if (usingSecureSocket) {
    794             throw new IllegalStateException("Already using a secure socket");
    795         }
    796
    797         if (LOG.isDebugEnabled()) {
    798             LOG.debug("Secure tunnel to " + this.hostName + ":" + this.portNumber);
    799         }
    800
    801         SecureProtocolSocketFactory socketFactory =
    802             (SecureProtocolSocketFactory) protocolInUse.getSocketFactory();
    803
    804         socket = socketFactory.createSocket(socket, hostName, portNumber, true);
    805         int sndBufSize = this.params.getSendBufferSize();
    806         if (sndBufSize >= 0) {
    807             socket.setSendBufferSize(sndBufSize);
    808         }
    809         int rcvBufSize = this.params.getReceiveBufferSize();
    810         if (rcvBufSize >= 0) {
    811             socket.setReceiveBufferSize(rcvBufSize);
    812         }
    813         int outbuffersize = socket.getSendBufferSize();
    814         if (outbuffersize > 2048) {
    815             outbuffersize = 2048;
    816         }
    817         int inbuffersize = socket.getReceiveBufferSize();
    818         if (inbuffersize > 2048) {
    819             inbuffersize = 2048;
    820         }
    821
    822         // START IA/HERITRIX change
    823         HttpRecorder httpRecorder = HttpRecorder.getHttpRecorder();
    824         if (httpRecorder == null) {
    825         inputStream = new BufferedInputStream(socket.getInputStream(), inbuffersize);
    826         outputStream = new BufferedOutputStream(socket.getOutputStream(), outbuffersize);
    827         } else {
    828             inputStream = httpRecorder.inputWrap((InputStream)
    829                     (new BufferedInputStream(socket.getInputStream(),
    830                     inbuffersize)));
    831             outputStream = httpRecorder.outputWrap((OutputStream)
    832                 (new BufferedOutputStream(socket.getOutputStream(),
    833                 outbuffersize)));
    834         }
    835         // END IA/HERITRIX change
    836
    837         usingSecureSocket = true;
    838         tunnelEstablished = true;
    839     }
    840
    841     /***
    842      * Indicates if the connection is completely transparent from end to end.
    843      *
    844      * @return true if conncetion is not proxied or tunneled through a transparent
    845      * proxy; false otherwise.
    846      */
    847     public boolean isTransparent() {
    848         return !isProxied() || tunnelEstablished;
    849     }
    850
    851     /***
    852      * Flushes the output request stream.  This method should be called to
    853      * ensure that data written to the request OutputStream is sent to the server.
    854      *
    855      * @throws IOException if an I/O problem occurs
    856      */
    857     public void flushRequestOutputStream() throws IOException {
    858         LOG.trace("enter HttpConnection.flushRequestOutputStream()");
    859         assertOpen();
    860         outputStream.flush();
    861     }
    862
    863     /***
    864      * Returns an {@link OutputStream} suitable for writing the request.
    865      *
    866      * @throws IllegalStateException if the connection is not open
    867      * @throws IOException if an I/O problem occurs
    868      * @return a stream to write the request to
    869      */
    870     public OutputStream getRequestOutputStream()
    871         throws IOException, IllegalStateException {
    872         LOG.trace("enter HttpConnection.getRequestOutputStream()");
    873         assertOpen();
    874         OutputStream out = this.outputStream;
    875         if (Wire.CONTENT_WIRE.enabled()) {
    876             out = new WireLogOutputStream(out, Wire.CONTENT_WIRE);
    877         }
    878         return out;
    879     }
    880
    881     /***
    882      * Return a {@link InputStream} suitable for reading the response.
    883      * @return InputStream The response input stream.
    884      * @throws IOException If an IO problem occurs
    885      * @throws IllegalStateException If the connection isn't open.
    886      */
    887     public InputStream getResponseInputStream()
    888         throws IOException, IllegalStateException {
    889         LOG.trace("enter HttpConnection.getResponseInputStream()");
    890         assertOpen();
    891         return inputStream;
    892     }
    893
    894     /***
    895      * Tests if input data avaialble. This method returns immediately
    896      * and does not perform any read operations on the input socket
    897      *
    898      * @return boolean <tt>true</tt> if input data is available,
    899      *                 <tt>false</tt> otherwise.
    900      *
    901      * @throws IOException If an IO problem occurs
    902      * @throws IllegalStateException If the connection isn't open.
    903      */
    904     public boolean isResponseAvailable()
    905         throws IOException {
    906         LOG.trace("enter HttpConnection.isResponseAvailable()");
    907         if (this.isOpen) {
    908             return this.inputStream.available() > 0;
    909         } else {
    910             return false;
    911         }
    912     }
    913
    914     /***
    915      * Tests if input data becomes available within the given period time in milliseconds.
    916      *
    917      * @param timeout The number milliseconds to wait for input data to become available
    918      * @return boolean <tt>true</tt> if input data is availble,
    919      *                 <tt>false</tt> otherwise.
    920      *
    921      * @throws IOException If an IO problem occurs
    922      * @throws IllegalStateException If the connection isn't open.
    923      */
    924     public boolean isResponseAvailable(int timeout)
    925         throws IOException {
    926         LOG.trace("enter HttpConnection.isResponseAvailable(int)");
    927         assertOpen();
    928         boolean result = false;
    929         if (this.inputStream.available() > 0) {
    930             result = true;
    931         } else {
    932             try {
    933                 this.socket.setSoTimeout(timeout);
    934                 inputStream.mark(1);
    935                 int byteRead = inputStream.read();
    936                 if (byteRead != -1) {
    937                     inputStream.reset();
    938                     LOG.debug("Input data available");
    939                     result = true;
    940                 } else {
    941                     LOG.debug("Input data not available");
    942                 }
    943             } catch (InterruptedIOException e) {
    944                 if (!ExceptionUtil.isSocketTimeoutException(e)) {
    945                     throw e;
    946                 }
    947                 if (LOG.isDebugEnabled()) {
    948                     LOG.debug("Input data not available after " + timeout + " ms");
    949                 }
    950             } finally {
    951                 try {
    952                     socket.setSoTimeout(this.params.getSoTimeout());
    953                 } catch (IOException ioe) {
    954                     LOG.debug("An error ocurred while resetting soTimeout, we will assume that"
    955                         + " no response is available.",
    956                         ioe);
    957                     result = false;
    958                 }
    959             }
    960         }
    961         return result;
    962     }
    963
    964     /***
    965      * Writes the specified bytes to the output stream.
    966      *
    967      * @param data the data to be written
    968      * @throws IllegalStateException if not connected
    969      * @throws IOException if an I/O problem occurs
    970      * @see #write(byte[],int,int)
    971      */
    972     public void write(byte[] data)
    973         throws IOException, IllegalStateException {
    974         LOG.trace("enter HttpConnection.write(byte[])");
    975         this.write(data, 0, data.length);
    976     }
    977
    978     /***
    979      * Writes <i>length</i> bytes in <i>data</i> starting at
    980      * <i>offset</i> to the output stream.
    981      *
    982      * The general contract for
    983      * write(b, off, len) is that some of the bytes in the array b are written
    984      * to the output stream in order; element b[off] is the first byte written
    985      * and b[off+len-1] is the last byte written by this operation.
    986      *
    987      * @param data array containing the data to be written.
    988      * @param offset the start offset in the data.
    989      * @param length the number of bytes to write.
    990      * @throws IllegalStateException if not connected
    991      * @throws IOException if an I/O problem occurs
    992      */
    993     public void write(byte[] data, int offset, int length)
    994         throws IOException, IllegalStateException {
    995         LOG.trace("enter HttpConnection.write(byte[], int, int)");
    996
    997         if (offset < 0) {
    998             throw new IllegalArgumentException("Array offset may not be negative");
    999         }
    1000         if (length < 0) {
    1001             throw new IllegalArgumentException("Array length may not be negative");
    1002         }
    1003         if (offset + length > data.length) {
    1004             throw new IllegalArgumentException("Given offset and length exceed the array length");
    1005         }
    1006         assertOpen();
    1007         this.outputStream.write(data, offset, length);
    1008     }
    1009
    1010     /***
    1011      * Writes the specified bytes, followed by <tt>"\r\n".getBytes()</tt> to the
    1012      * output stream.
    1013      *
    1014      * @param data the bytes to be written
    1015      * @throws IllegalStateException if the connection is not open
    1016      * @throws IOException if an I/O problem occurs
    1017      */
    1018     public void writeLine(byte[] data)
    1019         throws IOException, IllegalStateException {
    1020         LOG.trace("enter HttpConnection.writeLine(byte[])");
    1021         write(data);
    1022         writeLine();
    1023     }
    1024
    1025     /***
    1026      * Writes <tt>"\r\n".getBytes()</tt> to the output stream.
    1027      *
    1028      * @throws IllegalStateException if the connection is not open
    1029      * @throws IOException if an I/O problem occurs
    1030      */
    1031     public void writeLine()
    1032         throws IOException, IllegalStateException {
    1033         LOG.trace("enter HttpConnection.writeLine()");
    1034         write(CRLF);
    1035     }
    1036
    1037     /***
    1038      * @deprecated Use {@link #print(String, String)}
    1039      *
    1040      * Writes the specified String (as bytes) to the output stream.
    1041      *
    1042      * @param data the string to be written
    1043      * @throws IllegalStateException if the connection is not open
    1044      * @throws IOException if an I/O problem occurs
    1045      */
    1046     public void print(String data)
    1047         throws IOException, IllegalStateException {
    1048         LOG.trace("enter HttpConnection.print(String)");
    1049         write(EncodingUtil.getBytes(data, "ISO-8859-1"));
    1050     }
    1051
    1052     /***
    1053      * Writes the specified String (as bytes) to the output stream.
    1054      *
    1055      * @param data the string to be written
    1056      * @param charset the charset to use for writing the data
    1057      * @throws IllegalStateException if the connection is not open
    1058      * @throws IOException if an I/O problem occurs
    1059      *
    1060      * @since 3.0
    1061      */
    1062     public void print(String data, String charset)
    1063         throws IOException, IllegalStateException {
    1064         LOG.trace("enter HttpConnection.print(String)");
    1065         write(EncodingUtil.getBytes(data, charset));
    1066     }
    1067
    1068     /***
    1069      * @deprecated Use {@link #printLine(String, String)}
    1070      *
    1071      * Writes the specified String (as bytes), followed by
    1072      * <tt>"\r\n".getBytes()</tt> to the output stream.
    1073      *
    1074      * @param data the data to be written
    1075      * @throws IllegalStateException if the connection is not open
    1076      * @throws IOException if an I/O problem occurs
    1077      */
    1078     public void printLine(String data)
    1079         throws IOException, IllegalStateException {
    1080         LOG.trace("enter HttpConnection.printLine(String)");
    1081         writeLine(EncodingUtil.getBytes(data, "ISO-8859-1"));
    1082     }
    1083
    1084     /***
    1085      * Writes the specified String (as bytes), followed by
    1086      * <tt>"\r\n".getBytes()</tt> to the output stream.
    1087      *
    1088      * @param data the data to be written
    1089      * @param charset the charset to use for writing the data
    1090      * @throws IllegalStateException if the connection is not open
    1091      * @throws IOException if an I/O problem occurs
    1092      *
    1093      * @since 3.0
    1094      */
    1095     public void printLine(String data, String charset)
    1096         throws IOException, IllegalStateException {
    1097         LOG.trace("enter HttpConnection.printLine(String)");
    1098         writeLine(EncodingUtil.getBytes(data, charset));
    1099     }
    1100
    1101     /***
    1102      * Writes <tt>"\r\n".getBytes()</tt> to the output stream.
    1103      *
    1104      * @throws IllegalStateException if the connection is not open
    1105      * @throws IOException if an I/O problem occurs
    1106      */
    1107     public void printLine()
    1108         throws IOException, IllegalStateException {
    1109         LOG.trace("enter HttpConnection.printLine()");
    1110         writeLine();
    1111     }
    1112
    1113     /***
    1114      * Reads up to <tt>"\n"</tt> from the (unchunked) input stream.
    1115      * If the stream ends before the line terminator is found,
    1116      * the last part of the string will still be returned.
    1117      *
    1118      * @throws IllegalStateException if the connection is not open
    1119      * @throws IOException if an I/O problem occurs
    1120      * @return a line from the response
    1121      *
    1122      * @deprecated use #readLine(String)
    1123      */
    1124     public String readLine() throws IOException, IllegalStateException {
    1125         LOG.trace("enter HttpConnection.readLine()");
    1126
    1127         assertOpen();
    1128         return HttpParser.readLine(inputStream);
    1129     }
    1130
    1131     /***
    1132      * Reads up to <tt>"\n"</tt> from the (unchunked) input stream.
    1133      * If the stream ends before the line terminator is found,
    1134      * the last part of the string will still be returned.
    1135      *
    1136      * @param charset the charset to use for reading the data
    1137      *
    1138      * @throws IllegalStateException if the connection is not open
    1139      * @throws IOException if an I/O problem occurs
    1140      * @return a line from the response
    1141      *
    1142      * @since 3.0
    1143      */
    1144     public String readLine(final String charset) throws IOException, IllegalStateException {
    1145         LOG.trace("enter HttpConnection.readLine()");
    1146
    1147         assertOpen();
    1148         return HttpParser.readLine(inputStream, charset);
    1149     }
    1150
    1151     /***
    1152      * Attempts to shutdown the {@link Socket}'s output, via Socket.shutdownOutput()
    1153      * when running on JVM 1.3 or higher.
    1154      *
    1155      * @deprecated unused
    1156      */
    1157     public void shutdownOutput() {
    1158         LOG.trace("enter HttpConnection.shutdownOutput()");
    1159
    1160         try {
    1161             // Socket.shutdownOutput is a JDK 1.3
    1162             // method. We'll use reflection in case
    1163             // we're running in an older VM
    1164             Class[] paramsClasses = new Class[0];
    1165             Method shutdownOutput =
    1166                 socket.getClass().getMethod("shutdownOutput", paramsClasses);
    1167             Object[] params = new Object[0];
    1168             shutdownOutput.invoke(socket, params);
    1169         } catch (Exception ex) {
    1170             LOG.debug("Unexpected Exception caught", ex);
    1171             // Ignore, and hope everything goes right
    1172         }
    1173         // close output stream?
    1174     }
    1175
    1176     /***
    1177      * Closes the socket and streams.
    1178      */
    1179     public void close() {
    1180         LOG.trace("enter HttpConnection.close()");
    1181         closeSocketAndStreams();
    1182     }
    1183
    1184     /***
    1185      * Returns the httpConnectionManager.
    1186      * @return HttpConnectionManager
    1187      */
    1188     public HttpConnectionManager getHttpConnectionManager() {
    1189         return httpConnectionManager;
    1190     }
    1191
    1192     /***
    1193      * Sets the httpConnectionManager.
    1194      * @param httpConnectionManager The httpConnectionManager to set
    1195      */
    1196     public void setHttpConnectionManager(HttpConnectionManager httpConnectionManager) {
    1197         this.httpConnectionManager = httpConnectionManager;
    1198     }
    1199
    1200     /***
    1201      * Releases the connection. If the connection is locked or does not have a connection
    1202      * manager associated with it, this method has no effect. Note that it is completely safe
    1203      * to call this method multiple times.
    1204      */
    1205     public void releaseConnection() {
    1206         LOG.trace("enter HttpConnection.releaseConnection()");
    1207         if (locked) {
    1208             LOG.debug("Connection is locked.  Call to releaseConnection() ignored.");
    1209         } else if (httpConnectionManager != null) {
    1210             LOG.debug("Releasing connection back to connection manager.");
    1211             httpConnectionManager.releaseConnection(this);
    1212         } else {
    1213             LOG.warn("HttpConnectionManager is null.  Connection cannot be released.");
    1214         }
    1215     }
    1216
    1217     /***
    1218      * Tests if the connection is locked. Locked connections cannot be released.
    1219      * An attempt to release a locked connection will have no effect.
    1220      *
    1221      * @return <tt>true</tt> if the connection is locked, <tt>false</tt> otherwise.
    1222      *
    1223      * @since 3.0
    1224      */
    1225     protected boolean isLocked() {
    1226         return locked;
    1227     }
    1228
    1229     /***
    1230      * Locks or unlocks the connection. Locked connections cannot be released.
    1231      * An attempt to release a locked connection will have no effect.
    1232      *
    1233      * @param locked <tt>true</tt> to lock the connection, <tt>false</tt> to unlock
    1234      *  the connection.
    1235      *
    1236      * @since 3.0
    1237      */
    1238     protected void setLocked(boolean locked) {
    1239         this.locked = locked;
    1240     }
    1241     // ------------------------------------------------------ Protected Methods
    1242
    1243     /***
    1244      * Closes everything out.
    1245      */
    1246     protected void closeSocketAndStreams() {
    1247         LOG.trace("enter HttpConnection.closeSockedAndStreams()");
    1248
    1249         isOpen = false;
    1250
    1251         // no longer care about previous responses...
    1252         lastResponseInputStream = null;
    1253
    1254         if (null != outputStream) {
    1255             OutputStream temp = outputStream;
    1256             outputStream = null;
    1257             try {
    1258                 temp.close();
    1259             } catch (Exception ex) {
    1260                 LOG.debug("Exception caught when closing output", ex);
    1261                 // ignored
    1262             }
    1263         }
    1264
    1265         if (null != inputStream) {
    1266             InputStream temp = inputStream;
    1267             inputStream = null;
    1268             try {
    1269                 temp.close();
    1270             } catch (Exception ex) {
    1271                 LOG.debug("Exception caught when closing input", ex);
    1272                 // ignored
    1273             }
    1274         }
    1275
    1276         if (null != socket) {
    1277             Socket temp = socket;
    1278             socket = null;
    1279             try {
    1280                 temp.close();
    1281             } catch (Exception ex) {
    1282                 LOG.debug("Exception caught when closing socket", ex);
    1283                 // ignored
    1284             }
    1285         }
    1286
    1287         tunnelEstablished = false;
    1288         usingSecureSocket = false;
    1289     }
    1290
    1291     /***
    1292      * Throws an {@link IllegalStateException} if the connection is already open.
    1293      *
    1294      * @throws IllegalStateException if connected
    1295      */
    1296     protected void assertNotOpen() throws IllegalStateException {
    1297         if (isOpen) {
    1298             throw new IllegalStateException("Connection is open");
    1299         }
    1300     }
    1301
    1302     /***
    1303      * Throws an {@link IllegalStateException} if the connection is not open.
    1304      *
    1305      * @throws IllegalStateException if not connected
    1306      */
    1307     protected void assertOpen() throws IllegalStateException {
    1308         if (!isOpen) {
    1309             throw new IllegalStateException("Connection is not open");
    1310         }
    1311     }
    1312
    1313     /***
    1314      * Gets the socket's sendBufferSize.
    1315      *
    1316      * @return the size of the buffer for the socket OutputStream, -1 if the value
    1317      * has not been set and the socket has not been opened
    1318      *
    1319      * @throws SocketException if an error occurs while getting the socket value
    1320      *
    1321      * @see Socket#getSendBufferSize()
    1322      */
    1323     public int getSendBufferSize() throws SocketException {
    1324         if (socket == null) {
    1325             return -1;
    1326         } else {
    1327             return socket.getSendBufferSize();
    1328         }
    1329     }
    1330
    1331     /***
    1332      * Sets the socket's sendBufferSize.
    1333      *
    1334      * @param sendBufferSize the size to set for the socket OutputStream
    1335      *
    1336      * @throws SocketException if an error occurs while setting the socket value
    1337      *
    1338      * @see Socket#setSendBufferSize(int)
    1339      *
    1340      * @deprecated Use {@link HttpConnectionParams#setSendBufferSize(int)},
    1341      * {@link HttpConnection#getParams()}.
    1342      */
    1343     public void setSendBufferSize(int sendBufferSize) throws SocketException {
    1344         this.params.setSendBufferSize(sendBufferSize);
    1345     }
    1346
    1347     // ------------------------------------------------------- Static Variable
    1348
    1349     /*** <tt>"\r\n"</tt>, as bytes. */
    1350     private static final byte[] CRLF = new byte[] {(byte) 13, (byte) 10};
    1351
    1352     /*** Log object for this class. */
    1353     private static final Log LOG = LogFactory.getLog(HttpConnection.class);
    1354
    1355     // ----------------------------------------------------- Instance Variables
    1356
    1357     /*** My host. */
    1358     private String hostName = null;
    1359
    1360     /*** My port. */
    1361     private int portNumber = -1;
    1362
    1363     /*** My proxy host. */
    1364     private String proxyHostName = null;
    1365
    1366     /*** My proxy port. */
    1367     private int proxyPortNumber = -1;
    1368
    1369     /*** My client Socket. */
    1370     private Socket socket = null;
    1371
    1372     /*** My InputStream. */
    1373     private InputStream inputStream = null;
    1374
    1375     /*** My OutputStream. */
    1376     private OutputStream outputStream = null;
    1377
    1378     /*** An {@link InputStream} for the response to an individual request. */
    1379     private InputStream lastResponseInputStream = null;
    1380
    1381     /*** Whether or not the connection is connected. */
    1382     protected boolean isOpen = false;
    1383
    1384     /*** the protocol being used */
    1385     private Protocol protocolInUse;
    1386
    1387     /*** Collection of HTTP parameters associated with this HTTP connection*/
    1388     private HttpConnectionParams params = new HttpConnectionParams();
    1389
    1390     /*** flag to indicate if this connection can be released, if locked the connection cannot be
    1391      * released */
    1392     private boolean locked = false;
    1393
    1394     /*** Whether or not the socket is a secure one. */
    1395     private boolean usingSecureSocket = false;
    1396
    1397     /*** Whether the connection is open via a secure tunnel or not */
    1398     private boolean tunnelEstablished = false;
    1399
    1400     /*** the connection manager that created this connection or null */
    1401     private HttpConnectionManager httpConnectionManager;
    1402
    1403     /*** The local interface on which the connection is created, or null for the default */
    1404     private InetAddress localAddress;
    1405 }