View Javadoc

    1   /* WriterPool
    2    *
    3    * $Id: WriterPool.java 5479 2007-09-20 01:33:53Z gojomo $
    4    *
    5    * Created July 19th, 2006.
    6    *
    7    * Copyright (C) 2006 Internet Archive.
    8    *
    9    * This file is part of the Heritrix web crawler (crawler.archive.org).
    10   *
    11   * Heritrix is free software; you can redistribute it and/or modify
    12   * it under the terms of the GNU Lesser Public License as published by
    13   * the Free Software Foundation; either version 2.1 of the License, or
    14   * any later version.
    15   *
    16   * Heritrix is distributed in the hope that it will be useful,
    17   * but WITHOUT ANY WARRANTY; without even the implied warranty of
    18   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    19   * GNU Lesser Public License for more details.
    20   *
    21   * You should have received a copy of the GNU Lesser Public License
    22   * along with Heritrix; if not, write to the Free Software
    23   * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
    24   */
    25  package org.archive.io;
    26
    27  import java.io.File;
    28  import java.io.IOException;
    29  import java.util.NoSuchElementException;
    30  import java.util.concurrent.atomic.AtomicInteger;
    31  import java.util.logging.Level;
    32  import java.util.logging.Logger;
    33
    34  import org.apache.commons.pool.BasePoolableObjectFactory;
    35  import org.apache.commons.pool.impl.FairGenericObjectPool;
    36  import org.apache.commons.pool.impl.GenericObjectPool;
    37
    38  /***
    39   * Pool of Writers.
    40   *
    41   * Abstract. Override and pass in the Constructor a factory that creates
    42   * {@link WriterPoolMember} implementations.
    43   *
    44   * @author stack
    45   */
    46  public abstract class WriterPool {
    47      final Logger logger =  Logger.getLogger(this.getClass().getName());
    48
    49      /***
    50       * Used to generate unique filename sequences.
    51       */
    52      final private AtomicInteger serialNo;
    53
    54      /***
    55       * Don't enforce a maximum number of idle instances in pool.
    56       * To do so means GenericObjectPool will close files prematurely.
    57       */
    58      protected static final int NO_MAX_IDLE = -1;
    59
    60      /***
    61       * Retry getting a file on fail the below arbitrary amount of times.
    62       * This facility is not configurable.  If we fail this many times
    63       * getting a file, something is seriously wrong.
    64       */
    65      private final int arbitraryRetryMax = 10;
    66
    67  	/***
    68  	 * Default maximum active number of files in the pool.
    69  	 */
    70  	public static final int DEFAULT_MAX_ACTIVE = 5;
    71
    72  	/***
    73  	 * Maximum time to wait on a free file..
    74  	 */
    75  	public static final int DEFAULT_MAXIMUM_WAIT = 1000 * 60 * 5;
    76
    77      /***
    78       * Pool instance.
    79       */
    80      private GenericObjectPool pool = null;
    81
    82      /***
    83       * File settings.
    84       * Keep in data structure rather than as individual values.
    85       */
    86      private final WriterPoolSettings settings;
    87
    88      /***
    89       * Shutdown default constructor.
    90       */
    91      private WriterPool() {
    92      	this(null, null, null, -1, -1);
    93      }
    94
    95      /***
    96       * Constructor
    97       * @param serial  Used to generate unique filename sequences
    98       * @param factory Factory that knows how to make a {@link WriterPoolMember}.
    99       * @param settings Settings for this pool.
    100      * @param poolMaximumActive
    101      * @param poolMaximumWait
    102      */
    103     public WriterPool(final AtomicInteger serial,
    104     		final BasePoolableObjectFactory factory,
    105     		final WriterPoolSettings settings,
    106             final int poolMaximumActive, final int poolMaximumWait) {
    107         logger.info("Initial configuration:" +
    108                 " prefix=" + settings.getPrefix() +
    109                 ", suffix=" + settings.getSuffix() +
    110                 ", compress=" + settings.isCompressed() +
    111                 ", maxSize=" + settings.getMaxSize() +
    112                 ", maxActive=" + poolMaximumActive +
    113                 ", maxWait=" + poolMaximumWait);
    114         this.settings = settings;
    115         this.pool = new FairGenericObjectPool(factory, poolMaximumActive,
    116             GenericObjectPool.WHEN_EXHAUSTED_BLOCK, poolMaximumWait,
    117             NO_MAX_IDLE);
    118         this.serialNo = serial;
    119     }
    120
    121 	/***
    122 	 * Check out a {@link WriterPoolMember}.
    123 	 *
    124 	 * This method must be answered by a call to
    125 	 * {@link #returnFile(WriterPoolMember)} else pool starts leaking.
    126 	 *
    127 	 * @return Writer checked out of a pool of files.
    128 	 * @throws IOException Problem getting Writer from pool (Converted
    129 	 * from Exception to IOException so this pool can live as a good citizen
    130 	 * down in depths of ARCSocketFactory).
    131 	 * @throws NoSuchElementException If we time out waiting on a pool member.
    132 	 */
    133     public WriterPoolMember borrowFile()
    134     throws IOException {
    135         WriterPoolMember f = null;
    136         for (int i = 0; f == null; i++) {
    137             long waitStart = System.currentTimeMillis();
    138             try {
    139                 f = (WriterPoolMember)this.pool.borrowObject();
    140                 if (logger.getLevel() == Level.FINE) {
    141                     logger.fine("Borrowed " + f + " (Pool State: "
    142                         + getPoolState(waitStart) + ").");
    143                 }
    144             } catch (NoSuchElementException e) {
    145                 // Let this exception out. Unit test at least depends on it.
    146                 // Log current state of the pool.
    147                 logger.warning(e.getMessage() + ": Retry #" + i + " of "
    148                     + " max of " + arbitraryRetryMax
    149                     + ": NSEE Pool State: " + getPoolState(waitStart));
    150                 if (i >= arbitraryRetryMax) {
    151                     logger.log(Level.SEVERE,
    152                     	"maximum retries exceeded; rethrowing",e);
    153                     throw e;
    154                 }
    155             } catch (Exception e) {
    156                 // Convert.
    157                 logger.log(Level.SEVERE,"E Pool State: " +
    158                     getPoolState(waitStart), e);
    159                 throw new IOException("Failed getting writer from pool: " +
    160                     e.getMessage());
    161             }
    162         }
    163         return f;
    164     }
    165
    166 	/***
    167 	 * @param writer Writer to return to the pool.
    168 	 * @throws IOException Problem returning File to pool.
    169 	 */
    170     public void returnFile(WriterPoolMember writer)
    171     throws IOException {
    172         try {
    173             if (logger.getLevel() == Level.FINE) {
    174                 logger.fine("Returned " + writer);
    175             }
    176             this.pool.returnObject(writer);
    177         }
    178         catch(Exception e)
    179         {
    180             throw new IOException("Failed restoring writer to pool: " +
    181                     e.getMessage());
    182         }
    183     }
    184
    185     public void invalidateFile(WriterPoolMember f)
    186     throws IOException {
    187         try {
    188             this.pool.invalidateObject(f);
    189         } catch (Exception e) {
    190             // Convert exception.
    191             throw new IOException(e.getMessage());
    192         }
    193         // It'll have been closed.  Rename with an '.invalid' suffix so it
    194         // gets attention.
    195         File file = f.getFile();
    196         file.renameTo(new File(file.getAbsoluteFile() +
    197                 WriterPoolMember.INVALID_SUFFIX));
    198     }
    199
    200 	/***
    201 	 * @return Number of {@link WriterPoolMember}s checked out of pool.
    202 	 * @throws java.lang.UnsupportedOperationException
    203 	 */
    204     public int getNumActive()
    205     throws UnsupportedOperationException {
    206         return this.pool.getNumActive();
    207     }
    208
    209 	/***
    210 	 * @return Number of {@link WriterPoolMember} instances still in the pool.
    211 	 * @throws java.lang.UnsupportedOperationException
    212 	 */
    213     public int getNumIdle()
    214     throws UnsupportedOperationException {
    215         return this.pool.getNumIdle();
    216     }
    217
    218 	/***
    219 	 * Close all {@link WriterPoolMember}s in pool.
    220 	 */
    221     public void close() {
    222         this.pool.clear();
    223     }
    224
    225 	/***
    226 	 * @return Returns settings.
    227 	 */
    228     public WriterPoolSettings getSettings() {
    229         return this.settings;
    230     }
    231
    232     /***
    233      * @return State of the pool string
    234      */
    235     protected String getPoolState() {
    236         return getPoolState(-1);
    237     }
    238
    239     /***
    240      * @param startTime If we are passed a start time, we'll add difference
    241      * between it and now to end of string.  Pass -1 if don't want this
    242      * added to end of state string.
    243      * @return State of the pool string
    244      */
    245     protected String getPoolState(long startTime) {
    246         StringBuffer buffer = new StringBuffer("Active ");
    247         buffer.append(getNumActive());
    248         buffer.append(" of max ");
    249         buffer.append(this.pool.getMaxActive());
    250         buffer.append(", idle ");
    251         buffer.append(this.pool.getNumIdle());
    252         if (startTime != -1) {
    253             buffer.append(", time ");
    254             buffer.append(System.currentTimeMillis() - startTime);
    255             buffer.append("ms of max ");
    256             buffer.append(this.pool.getMaxWait());
    257             buffer.append("ms");
    258         }
    259         return buffer.toString();
    260     }
    261
    262     /***
    263      * Returns the atomic integer used to generate serial numbers
    264      * for files.
    265      *
    266      * @return  the serial number generator
    267      */
    268     public AtomicInteger getSerialNo() {
    269         return serialNo;
    270     }
    271 }