1   /* ARCWriterPoolTest
2    *
3    * $Id: ARCWriterPoolTest.java 5029 2007-03-29 23:53:50Z gojomo $
4    *
5    * Created on Jan 22, 2004
6    *
7    * Copyright (C) 2004 Internet Archive.
8    *
9    * This file is part of the Heritrix web crawler (crawler.archive.org).
10   *
11   * Heritrix is free software; you can redistribute it and/or modify
12   * it under the terms of the GNU Lesser Public License as published by
13   * the Free Software Foundation; either version 2.1 of the License, or
14   * any later version.
15   *
16   * Heritrix is distributed in the hope that it will be useful,
17   * but WITHOUT ANY WARRANTY; without even the implied warranty of
18   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19   * GNU Lesser Public License for more details.
20   *
21   * You should have received a copy of the GNU Lesser Public License
22   * along with Heritrix; if not, write to the Free Software
23   * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
24   */
25  package org.archive.io.arc;
26  
27  import java.io.ByteArrayOutputStream;
28  import java.io.File;
29  import java.util.Arrays;
30  import java.util.Date;
31  import java.util.List;
32  import java.util.NoSuchElementException;
33  
34  import org.archive.io.WriterPoolMember;
35  import org.archive.io.WriterPool;
36  import org.archive.io.WriterPoolSettings;
37  import org.archive.util.TmpDirTestCase;
38  
39  
40  /***
41   * Test ARCWriterPool
42   */
43  public class ARCWriterPoolTest extends TmpDirTestCase {
44      private static final String PREFIX = "TEST";
45      
46      public void testARCWriterPool()
47      throws Exception {
48          final int MAX_ACTIVE = 3;
49          final int MAX_WAIT_MILLISECONDS = 100;
50          cleanUpOldFiles(PREFIX);
51          WriterPool pool = new ARCWriterPool(getSettings(true),
52              MAX_ACTIVE, MAX_WAIT_MILLISECONDS);
53          WriterPoolMember [] writers = new WriterPoolMember[MAX_ACTIVE];
54          final String CONTENT = "Any old content";
55          ByteArrayOutputStream baos = new ByteArrayOutputStream();
56          baos.write(CONTENT.getBytes());
57          for (int i = 0; i < MAX_ACTIVE; i++) {
58              writers[i] = pool.borrowFile();
59              assertEquals("Number active", i + 1, pool.getNumActive());
60              ((ARCWriter)writers[i]).write("http://one.two.three", "no-type",
61              	"0.0.0.0", 1234567890, CONTENT.length(), baos);
62          }
63  
64          // Pool is maxed out.  Try and get a new ARCWriter.  We'll block for
65          // MAX_WAIT_MILLISECONDS.  Should get exception.
66          long start = (new Date()).getTime();
67          boolean isException = false;
68          try {
69              pool.borrowFile();
70          } catch(NoSuchElementException e) {
71              isException = true;
72              long end = (new Date()).getTime();
73              // This test can fail on a loaded machine if the wait period is
74              // only MAX_WAIT_MILLISECONDS.  Up the time to wait.
75              final int WAIT = MAX_WAIT_MILLISECONDS * 100;
76              if ((end - start) > (WAIT)) {
77                  fail("More than " + MAX_WAIT_MILLISECONDS + " elapsed: "
78                      + WAIT);
79              }
80          }
81          assertTrue("Did not get NoSuchElementException", isException);
82  
83          for (int i = (MAX_ACTIVE - 1); i >= 0; i--) {
84              pool.returnFile(writers[i]);
85              assertEquals("Number active", i, pool.getNumActive());
86              assertEquals("Number idle", MAX_ACTIVE - pool.getNumActive(),
87                      pool.getNumIdle());
88          }
89          pool.close();
90      }
91      
92      public void testInvalidate() throws Exception {
93          final int MAX_ACTIVE = 3;
94          final int MAX_WAIT_MILLISECONDS = 100;
95          cleanUpOldFiles(PREFIX);
96          WriterPool pool = new ARCWriterPool(getSettings(true),
97              MAX_ACTIVE, MAX_WAIT_MILLISECONDS);
98          WriterPoolMember [] writers = new WriterPoolMember[MAX_ACTIVE];
99          final String CONTENT = "Any old content";
100         ByteArrayOutputStream baos = new ByteArrayOutputStream();
101         baos.write(CONTENT.getBytes());
102         for (int i = 0; i < MAX_ACTIVE; i++) {
103             writers[i] = pool.borrowFile();
104             assertEquals("Number active", i + 1, pool.getNumActive());
105             ((ARCWriter)writers[i]).write("http://one.two.three", "no-type",
106             	"0.0.0.0", 1234567890, CONTENT.length(), baos);
107         }
108      
109         WriterPoolMember writer2Invalidate = writers[pool.getNumActive() - 1];
110         writers[pool.getNumActive() - 1] = null;
111         pool.invalidateFile(writer2Invalidate);
112         for (int i = 0; i < (MAX_ACTIVE - 1); i++) {
113             if (writers[i] == null) {
114                 continue;
115             }
116             pool.returnFile(writers[i]);
117         }
118         
119         for (int i = 0; i < MAX_ACTIVE; i++) {
120             writers[i] = pool.borrowFile();
121             assertEquals("Number active", i + 1, pool.getNumActive());
122             ((ARCWriter)writers[i]).write("http://one.two.three", "no-type",
123             	"0.0.0.0", 1234567890, CONTENT.length(), baos);
124         }
125         for (int i = (MAX_ACTIVE - 1); i >= 0; i--) {
126             pool.returnFile(writers[i]);
127             assertEquals("Number active", i, pool.getNumActive());
128             assertEquals("Number idle", MAX_ACTIVE - pool.getNumActive(),
129                     pool.getNumIdle());
130         }
131         pool.close();
132     }
133     
134     private WriterPoolSettings getSettings(final boolean isCompressed) {
135         return new WriterPoolSettings() {
136             public long getMaxSize() {
137                 return ARCConstants.DEFAULT_MAX_ARC_FILE_SIZE;
138             }
139             
140             public String getPrefix() {
141                 return PREFIX;
142             }
143             
144             public String getSuffix() {
145                 return "";
146             }
147             
148             public List<File> getOutputDirs() {
149                 File [] files = {getTmpDir()};
150                 return Arrays.asList(files);
151             }
152             
153             public boolean isCompressed() {
154                 return isCompressed;
155             }
156             
157             public List getMetadata() {
158             	return null;
159             }
160         };
161     }
162 }