    1   /* IoUtils
    2    *
    3    * Created on Dec 8, 2004
    4    *
    5    * Copyright (C) 2004 Internet Archive.
    6    *
    7    * This file is part of the Heritrix web crawler (crawler.archive.org).
    8    *
    9    * Heritrix is free software; you can redistribute it and/or modify
    10   * it under the terms of the GNU Lesser Public License as published by
    11   * the Free Software Foundation; either version 2.1 of the License, or
    12   * any later version.
    13   *
    14   * Heritrix is distributed in the hope that it will be useful,
    15   * but WITHOUT ANY WARRANTY; without even the implied warranty of
    16   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    17   * GNU Lesser Public License for more details.
    18   *
    19   * You should have received a copy of the GNU Lesser Public License
    20   * along with Heritrix; if not, write to the Free Software
    21   * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
    22   */
    23  package org.archive.util;
    24
    25  import it.unimi.dsi.fastutil.io.FastBufferedOutputStream;
    26
    27  import java.io.BufferedInputStream;
    28  import java.io.BufferedOutputStream;
    29  import java.io.ByteArrayInputStream;
    30  import java.io.ByteArrayOutputStream;
    31  import java.io.EOFException;
    32  import java.io.File;
    33  import java.io.FileInputStream;
    34  import java.io.FileOutputStream;
    35  import java.io.IOException;
    36  import java.io.InputStream;
    37  import java.io.ObjectInputStream;
    38  import java.io.ObjectOutputStream;
    39  import java.io.OutputStream;
    40  import java.nio.charset.Charset;
    41  import java.util.Iterator;
    42  import java.util.List;
    43  import java.util.logging.Level;
    44  import java.util.logging.Logger;
    45
    46  /***
    47   * I/O Utility methods.
    48   * @author stack
    49   * @version $Date: 2007-02-20 23:25:20 +0000 (Tue, 20 Feb 2007) $, $Revision: 4919 $
    50   */
    51  public class IoUtils {
    52      protected static Logger logger =
    53          Logger.getLogger(IoUtils.class.getName());
    54
    55      /***
    56       * @param file File to operate on.
    57       * @return Path suitable for use getting resources off the CLASSPATH
    58       * (CLASSPATH resources always use '/' as path separator, even on
    59       * windows).
    60       */
    61      public static String getClasspathPath(File file) {
    62          String path = file.getPath();
    63          if (File.separatorChar != '/') {
    64              // OK.  We're probably on a windows system. Strip
    65              // drive if its present and convert '\' to '/'.
    66              path = path.replace(File.separatorChar, '/');
    67              int index = path.indexOf(':');
    68              if (index > 0 && index < 3) {
    69                  path = path.substring(index + 1);
    70              }
    71          }
    72          return path;
    73      }
    74
    75      /***
    76       * Ensure writeable directory.
    77       *
    78       * If doesn't exist, we attempt creation.
    79       *
    80       * @param dir Directory to test for exitence and is writeable.
    81       *
    82       * @return The passed <code>dir</code>.
    83       *
    84       * @exception IOException If passed directory does not exist and is not
    85       * createable, or directory is not writeable or is not a directory.
    86       */
    87      public static File ensureWriteableDirectory(String dir)
    88      throws IOException {
    89          return ensureWriteableDirectory(new File(dir));
    90      }
    91
    92      /***
    93       * Ensure writeable directories.
    94       *
    95       * If doesn't exist, we attempt creation.
    96       *
    97       * @param dirs List of Files to test.
    98       *
    99       * @return The passed <code>dirs</code>.
    100      *
    101      * @exception IOException If passed directory does not exist and is not
    102      * createable, or directory is not writeable or is not a directory.
    103      */
    104     public static List ensureWriteableDirectory(List<File> dirs)
    105     throws IOException {
    106         for (Iterator<File> i = dirs.iterator(); i.hasNext();) {
    107              ensureWriteableDirectory(i.next());
    108         }
    109         return dirs;
    110     }
    111
    112     /***
    113      * Ensure writeable directory.
    114      *
    115      * If doesn't exist, we attempt creation.
    116      *
    117      * @param dir Directory to test for exitence and is writeable.
    118      *
    119      * @return The passed <code>dir</code>.
    120      *
    121      * @exception IOException If passed directory does not exist and is not
    122      * createable, or directory is not writeable or is not a directory.
    123      */
    124     public static File ensureWriteableDirectory(File dir)
    125     throws IOException {
    126         if (!dir.exists()) {
    127             dir.mkdirs();
    128         } else {
    129             if (!dir.canWrite()) {
    130                 throw new IOException("Dir " + dir.getAbsolutePath() +
    131                     " not writeable.");
    132             } else if (!dir.isDirectory()) {
    133                 throw new IOException("Dir " + dir.getAbsolutePath() +
    134                     " is not a directory.");
    135             }
    136         }
    137
    138         return dir;
    139     }
    140
    141     /***
    142      * Read the entire stream to EOF, returning what's read as a String.
    143      *
    144      * @param inputStream
    145      * @return String of the whole inputStream's contents
    146      * @throws IOException
    147      */
    148     public static String readFullyAsString(InputStream inputStream)
    149     throws IOException {
    150         StringBuffer sb = new StringBuffer();
    151         int c;
    152         while((c = inputStream.read()) > -1) {
    153             sb.append((char)c);
    154         }
    155         return sb.toString();
    156     }
    157
    158     /***
    159      * Read the entire stream to EOF into the passed file.
    160      * @param is
    161      * @param toFile File to read into .
    162      * @throws IOException
    163      * @throws IOException
    164      */
    165     public static void readFullyToFile(InputStream is,
    166             File toFile) throws IOException {
    167         readFullyToFile(is, toFile, new byte[4096]);
    168     }
    169
    170     /***
    171      * Read the entire stream to EOF into the passed file.
    172      * Closes <code>is</code> when done or if an exception.
    173      * @param is Stream to read.
    174      * @param toFile File to read into .
    175      * @param buffer Buffer to use reading.
    176      * @return Count of bytes read.
    177      * @throws IOException
    178      */
    179     public static long readFullyToFile(final InputStream is, final File toFile,
    180             final byte [] buffer)
    181     throws IOException {
    182         long totalcount = -1;
    183         OutputStream os =
    184             new FastBufferedOutputStream(new FileOutputStream(toFile));
    185         InputStream localIs = (is instanceof BufferedInputStream)?
    186             is: new BufferedInputStream(is);
    187         try {
    188             for (int count = -1;
    189                 (count = localIs.read(buffer, 0, buffer.length)) != -1;
    190                     totalcount += count) {
    191                 os.write(buffer, 0, count);
    192             }
    193         } finally {
    194             os.close();
    195             if (localIs != null) {
    196                 localIs.close();
    197             }
    198         }
    199         return totalcount;
    200     }
    201
    202     /***
    203      * Wrap generic Throwable as a checked IOException
    204      * @param e wrapped exception
    205      * @return IOException
    206      */
    207     public static IOException wrapAsIOException(Throwable e) {
    208         IOException ioe = new IOException(e.toString());
    209         ioe.initCause(e);
    210         return ioe;
    211     }
    212
    213
    214     public static void readFully(InputStream input, byte[] buf)
    215     throws IOException {
    216         int max = buf.length;
    217         int ofs = 0;
    218         while (ofs < max) {
    219             int l = input.read(buf, ofs, max - ofs);
    220             if (l == 0) {
    221                 throw new EOFException();
    222             }
    223             ofs += l;
    224         }
    225     }
    226
    227     /***
    228      * Return the maximum number of bytes per character in the named
    229      * encoding, or 0 if encoding is invalid or unsupported.
    230      *
    231      * @param encoding Encoding to consider.  For now, should be java
    232      * canonical name for the encoding.
    233      *
    234      * @return True if multibyte encoding.
    235      */
    236     public static float encodingMaxBytesPerChar(String encoding) {
    237         boolean isMultibyte = false;
    238         final Charset cs;
    239         try {
    240             if (encoding != null && encoding.length() > 0) {
    241                 cs = Charset.forName(encoding);
    242                 if(cs.canEncode()) {
    243                     return cs.newEncoder().maxBytesPerChar();
    244                 } else {
    245                     logger.info("Encoding not fully supported: " + encoding
    246                             + ".  Defaulting to single byte.");
    247                 }
    248             }
    249         } catch (IllegalArgumentException e) {
    250             // Unsupported encoding
    251             logger.log(Level.INFO,"Illegal encoding name: " + encoding,e);
    252         }
    253
    254         logger.fine("Encoding " + encoding + " is multibyte: "
    255             + ((isMultibyte) ? Boolean.TRUE : Boolean.FALSE));
    256         // default: return 0
    257         return 0;
    258     }
    259
    260     /***
    261      * Utility method to serialize an object to the given File.
    262      *
    263      * @param object Object to serialize
    264      * @param file File to receive serialized copy
    265      * @throws IOException
    266      */
    267     public static void serializeToFile(Object object, File file) throws IOException {
    268         ObjectOutputStream oos = new ObjectOutputStream(new BufferedOutputStream(new FileOutputStream(file)));
    269         oos.writeObject(object);
    270         oos.close();
    271     }
    272
    273     /***
    274      * Utility method to deserialize an Object from given File.
    275      *
    276      * @param file File source
    277      * @return deserialized Object
    278      * @throws IOException
    279      */
    280     public static Object deserializeFromFile(File file) throws IOException {
    281         ObjectInputStream ois = new ObjectInputStream(new BufferedInputStream(new FileInputStream(file)));
    282         Object object;
    283         try {
    284             object = ois.readObject();
    285         } catch (ClassNotFoundException e) {
    286             // TODO Auto-generated catch block
    287             throw new RuntimeException(e);
    288         }
    289         ois.close();
    290         return object;
    291     }
    292
    293     /***
    294      * Utility method to serialize Object to byte[].
    295      *
    296      * @param object Object to be serialized
    297      * @return byte[] serialized form
    298      */
    299     public static byte[] serializeToByteArray(Object object) {
    300         ByteArrayOutputStream baos = new ByteArrayOutputStream();
    301         try {
    302             ObjectOutputStream oos = new ObjectOutputStream(baos);
    303             oos.writeObject(object);
    304             oos.close();
    305         } catch (IOException e) {
    306             // shouldn't be possible
    307             throw new RuntimeException(e);
    308         }
    309         return baos.toByteArray();
    310     }
    311
    312     /***
    313      * Utility method to deserialize Object from  byte[].
    314      *
    315      * @param in byte[] source
    316      * @return Object deserialized
    317      */
    318     public static Object deserializeFromByteArray(byte[] in) {
    319         Object object;
    320         try {
    321             ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(in));
    322             try {
    323                 object = ois.readObject();
    324             } catch (ClassNotFoundException e) {
    325                 // TODO Auto-generated catch block
    326                 throw new RuntimeException(e);
    327             }
    328             ois.close();
    329         } catch (IOException e) {
    330             // shouldn't be possible
    331             throw new RuntimeException(e);
    332         }
    333         return object;
    334     }
    335 }