    1   /* $Id: Handler.java 4902 2007-02-16 00:07:40Z stack-sf $
    2    *
    3    * Created October 28th, 2006
    4    *
    5    * Copyright (C) 2006 Internet Archive.
    6    *
    7    * This file is part of the Heritrix web crawler (crawler.archive.org).
    8    *
    9    * Heritrix is free software; you can redistribute it and/or modify
    10   * it under the terms of the GNU Lesser Public License as published by
    11   * the Free Software Foundation; either version 2.1 of the License, or
    12   * any later version.
    13   *
    14   * Heritrix is distributed in the hope that it will be useful,
    15   * but WITHOUT ANY WARRANTY; without even the implied warranty of
    16   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    17   * GNU Lesser Public License for more details.
    18   *
    19   * You should have received a copy of the GNU Lesser Public License
    20   * along with Heritrix; if not, write to the Free Software
    21   * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
    22   */
    23  package org.archive.net.s3;
    24
    25  import java.io.IOException;
    26  import java.io.InputStream;
    27  import java.io.PrintStream;
    28  import java.net.HttpURLConnection;
    29  import java.net.URL;
    30  import java.net.URLConnection;
    31  import java.net.URLStreamHandler;
    32  import java.text.SimpleDateFormat;
    33  import java.util.Date;
    34  import java.util.Locale;
    35  import java.util.TimeZone;
    36
    37  import org.jets3t.service.S3ServiceException;
    38  import org.jets3t.service.impl.rest.httpclient.RestS3Service;
    39  import org.jets3t.service.model.S3Bucket;
    40  import org.jets3t.service.model.S3Object;
    41  import org.jets3t.service.security.AWSCredentials;
    42
    43  /***
    44   * A protocol handler for an s3 scheme. Takes URLs of the form:
    45   * <code>s3://aws.access.key.id:aws.access.key.secret@BUCKET/PATH</code> (Same
    46   * as in hadoop).
    47   *
    48   * @author stack
    49   */
    50  public class Handler extends URLStreamHandler {
    51      protected URLConnection openConnection(URL u)
    52      throws IOException {
    53          // This looking for accessKey id and accessKey secret code is based
    54          // on code from hadoop S3.
    55          String accessKey = null;
    56          String secretAccessKey = null;
    57          String userInfo = u.getUserInfo();
    58          if (userInfo != null) {
    59              int index = userInfo.indexOf(':');
    60              if (index != -1) {
    61                accessKey = userInfo.substring(0, index);
    62                secretAccessKey = userInfo.substring(index + 1);
    63              } else {
    64                accessKey = userInfo;
    65              }
    66          }
    67          if (accessKey == null) {
    68            accessKey = System.getProperty("aws.access.key.id");
    69          }
    70          if (secretAccessKey == null) {
    71            secretAccessKey = System.getProperty("aws.access.key.secret");
    72          }
    73          if (accessKey == null && secretAccessKey == null) {
    74            throw new IllegalArgumentException("AWS " +
    75                  "Access Key ID and Secret Access Key " +
    76                  "must be specified as the username " +
    77                  "or password (respectively) of a s3 URL, " +
    78                  "or by setting the " +
    79                  "aws.access.key.id or " +
    80                  "aws.access.key.secret properties (respectively).");
    81          } else if (accessKey == null) {
    82            throw new IllegalArgumentException("AWS " +
    83                  "Access Key ID must be specified " +
    84                  "as the username of a s3 URL, or by setting the " +
    85                  "aws.access.key.id property.");
    86          } else if (secretAccessKey == null) {
    87            throw new IllegalArgumentException("AWS " +
    88                  "Secret Access Key must be specified " +
    89                  "as the password of a s3 URL, or by setting the " +
    90                  "aws.access.key.secret property.");
    91          }
    92
    93          RestS3Service s3Service;
    94          try {
    95              s3Service = new RestS3Service(
    96                  new AWSCredentials(accessKey, secretAccessKey));
    97          } catch (S3ServiceException e) {
    98              e.printStackTrace();
    99              throw new IOException(e.toString());
    100         }
    101         InputStream is = null;
    102         try {
    103             // This opens the stream to the bucket/key object.
    104             S3Object s3obj = s3Service.getObject(new S3Bucket(u.getHost()),
    105                 u.getPath().substring(1) /* Skip starting '/' character */);
    106             is = s3obj.getDataInputStream();
    107         } catch (S3ServiceException e) {
    108             e.printStackTrace();
    109             throw new IOException(e.toString());
    110         }
    111
    112         final InputStream inputStream = is;
    113         return new URLConnection(u) {
    114             private InputStream is = inputStream;
    115             @Override
    116             public InputStream getInputStream() throws IOException {
    117                 return this.is;
    118             }
    119             @Override
    120             public void connect() throws IOException {
    121                 // Nothing to do. When we give back this object, we're
    122                 // connected.
    123             }
    124         };
    125     }
    126
    127     /***
    128      * Main dumps rsync file to STDOUT.
    129      * @param args
    130      * @throws IOException
    131      */
    132     public static void main(String[] args)
    133     throws IOException {
    134         if (args.length != 1) {
    135             System.out.println("Usage: java " +
    136                 "org.archive.net.s3.Handler " +
    137                 "s3://AWS_ACCESS_KEY_ID:AWS_ACCESS_KEY_SECRET@BUCKET/KEY");
    138             System.exit(1);
    139         }
    140         URL u = new URL(args[0]);
    141         URLConnection connect = u.openConnection();
    142         // Write download to stdout.
    143         final int bufferlength = 4096;
    144         byte [] buffer = new byte [bufferlength];
    145         InputStream is = connect.getInputStream();
    146         try {
    147             for (int count = -1;
    148                     (count = is.read(buffer, 0, bufferlength)) != -1;) {
    149                 System.out.write(buffer, 0, count);
    150             }
    151             System.out.flush();
    152         } finally {
    153             is.close();
    154         }
    155     }
    156 }