View Javadoc

    1   /* ARCWriterPoolTest
    2    *
    3    * $Id: ARCWriterPoolTest.java 5029 2007-03-29 23:53:50Z gojomo $
    4    *
    5    * Created on Jan 22, 2004
    6    *
    7    * Copyright (C) 2004 Internet Archive.
    8    *
    9    * This file is part of the Heritrix web crawler (crawler.archive.org).
    10   *
    11   * Heritrix is free software; you can redistribute it and/or modify
    12   * it under the terms of the GNU Lesser Public License as published by
    13   * the Free Software Foundation; either version 2.1 of the License, or
    14   * any later version.
    15   *
    16   * Heritrix is distributed in the hope that it will be useful,
    17   * but WITHOUT ANY WARRANTY; without even the implied warranty of
    18   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    19   * GNU Lesser Public License for more details.
    20   *
    21   * You should have received a copy of the GNU Lesser Public License
    22   * along with Heritrix; if not, write to the Free Software
    23   * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
    24   */
    25  package org.archive.io.arc;
    26
    27  import java.io.ByteArrayOutputStream;
    28  import java.io.File;
    29  import java.util.Arrays;
    30  import java.util.Date;
    31  import java.util.List;
    32  import java.util.NoSuchElementException;
    33
    34  import org.archive.io.WriterPoolMember;
    35  import org.archive.io.WriterPool;
    36  import org.archive.io.WriterPoolSettings;
    37  import org.archive.util.TmpDirTestCase;
    38
    39
    40  /***
    41   * Test ARCWriterPool
    42   */
    43  public class ARCWriterPoolTest extends TmpDirTestCase {
    44      private static final String PREFIX = "TEST";
    45
    46      public void testARCWriterPool()
    47      throws Exception {
    48          final int MAX_ACTIVE = 3;
    49          final int MAX_WAIT_MILLISECONDS = 100;
    50          cleanUpOldFiles(PREFIX);
    51          WriterPool pool = new ARCWriterPool(getSettings(true),
    52              MAX_ACTIVE, MAX_WAIT_MILLISECONDS);
    53          WriterPoolMember [] writers = new WriterPoolMember[MAX_ACTIVE];
    54          final String CONTENT = "Any old content";
    55          ByteArrayOutputStream baos = new ByteArrayOutputStream();
    56          baos.write(CONTENT.getBytes());
    57          for (int i = 0; i < MAX_ACTIVE; i++) {
    58              writers[i] = pool.borrowFile();
    59              assertEquals("Number active", i + 1, pool.getNumActive());
    60              ((ARCWriter)writers[i]).write("http://one.two.three", "no-type",
    61              	"0.0.0.0", 1234567890, CONTENT.length(), baos);
    62          }
    63
    64          // Pool is maxed out.  Try and get a new ARCWriter.  We'll block for
    65          // MAX_WAIT_MILLISECONDS.  Should get exception.
    66          long start = (new Date()).getTime();
    67          boolean isException = false;
    68          try {
    69              pool.borrowFile();
    70          } catch(NoSuchElementException e) {
    71              isException = true;
    72              long end = (new Date()).getTime();
    73              // This test can fail on a loaded machine if the wait period is
    74              // only MAX_WAIT_MILLISECONDS.  Up the time to wait.
    75              final int WAIT = MAX_WAIT_MILLISECONDS * 100;
    76              if ((end - start) > (WAIT)) {
    77                  fail("More than " + MAX_WAIT_MILLISECONDS + " elapsed: "
    78                      + WAIT);
    79              }
    80          }
    81          assertTrue("Did not get NoSuchElementException", isException);
    82
    83          for (int i = (MAX_ACTIVE - 1); i >= 0; i--) {
    84              pool.returnFile(writers[i]);
    85              assertEquals("Number active", i, pool.getNumActive());
    86              assertEquals("Number idle", MAX_ACTIVE - pool.getNumActive(),
    87                      pool.getNumIdle());
    88          }
    89          pool.close();
    90      }
    91
    92      public void testInvalidate() throws Exception {
    93          final int MAX_ACTIVE = 3;
    94          final int MAX_WAIT_MILLISECONDS = 100;
    95          cleanUpOldFiles(PREFIX);
    96          WriterPool pool = new ARCWriterPool(getSettings(true),
    97              MAX_ACTIVE, MAX_WAIT_MILLISECONDS);
    98          WriterPoolMember [] writers = new WriterPoolMember[MAX_ACTIVE];
    99          final String CONTENT = "Any old content";
    100         ByteArrayOutputStream baos = new ByteArrayOutputStream();
    101         baos.write(CONTENT.getBytes());
    102         for (int i = 0; i < MAX_ACTIVE; i++) {
    103             writers[i] = pool.borrowFile();
    104             assertEquals("Number active", i + 1, pool.getNumActive());
    105             ((ARCWriter)writers[i]).write("http://one.two.three", "no-type",
    106             	"0.0.0.0", 1234567890, CONTENT.length(), baos);
    107         }
    108
    109         WriterPoolMember writer2Invalidate = writers[pool.getNumActive() - 1];
    110         writers[pool.getNumActive() - 1] = null;
    111         pool.invalidateFile(writer2Invalidate);
    112         for (int i = 0; i < (MAX_ACTIVE - 1); i++) {
    113             if (writers[i] == null) {
    114                 continue;
    115             }
    116             pool.returnFile(writers[i]);
    117         }
    118
    119         for (int i = 0; i < MAX_ACTIVE; i++) {
    120             writers[i] = pool.borrowFile();
    121             assertEquals("Number active", i + 1, pool.getNumActive());
    122             ((ARCWriter)writers[i]).write("http://one.two.three", "no-type",
    123             	"0.0.0.0", 1234567890, CONTENT.length(), baos);
    124         }
    125         for (int i = (MAX_ACTIVE - 1); i >= 0; i--) {
    126             pool.returnFile(writers[i]);
    127             assertEquals("Number active", i, pool.getNumActive());
    128             assertEquals("Number idle", MAX_ACTIVE - pool.getNumActive(),
    129                     pool.getNumIdle());
    130         }
    131         pool.close();
    132     }
    133
    134     private WriterPoolSettings getSettings(final boolean isCompressed) {
    135         return new WriterPoolSettings() {
    136             public long getMaxSize() {
    137                 return ARCConstants.DEFAULT_MAX_ARC_FILE_SIZE;
    138             }
    139
    140             public String getPrefix() {
    141                 return PREFIX;
    142             }
    143
    144             public String getSuffix() {
    145                 return "";
    146             }
    147
    148             public List<File> getOutputDirs() {
    149                 File [] files = {getTmpDir()};
    150                 return Arrays.asList(files);
    151             }
    152
    153             public boolean isCompressed() {
    154                 return isCompressed;
    155             }
    156
    157             public List getMetadata() {
    158             	return null;
    159             }
    160         };
    161     }
    162 }