View Javadoc

1   /* $Id: Handler.java 4902 2007-02-16 00:07:40Z stack-sf $
2    *
3    * Created October 28th, 2006
4    *
5    * Copyright (C) 2006 Internet Archive.
6    *
7    * This file is part of the Heritrix web crawler (crawler.archive.org).
8    *
9    * Heritrix is free software; you can redistribute it and/or modify
10   * it under the terms of the GNU Lesser Public License as published by
11   * the Free Software Foundation; either version 2.1 of the License, or
12   * any later version.
13   *
14   * Heritrix is distributed in the hope that it will be useful,
15   * but WITHOUT ANY WARRANTY; without even the implied warranty of
16   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17   * GNU Lesser Public License for more details.
18   *
19   * You should have received a copy of the GNU Lesser Public License
20   * along with Heritrix; if not, write to the Free Software
21   * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
22   */
23  package org.archive.net.s3;
24  
25  import java.io.IOException;
26  import java.io.InputStream;
27  import java.io.PrintStream;
28  import java.net.HttpURLConnection;
29  import java.net.URL;
30  import java.net.URLConnection;
31  import java.net.URLStreamHandler;
32  import java.text.SimpleDateFormat;
33  import java.util.Date;
34  import java.util.Locale;
35  import java.util.TimeZone;
36  
37  import org.jets3t.service.S3ServiceException;
38  import org.jets3t.service.impl.rest.httpclient.RestS3Service;
39  import org.jets3t.service.model.S3Bucket;
40  import org.jets3t.service.model.S3Object;
41  import org.jets3t.service.security.AWSCredentials;
42  
43  /***
44   * A protocol handler for an s3 scheme. Takes URLs of the form:
45   * <code>s3://aws.access.key.id:aws.access.key.secret@BUCKET/PATH</code> (Same
46   * as in hadoop).
47   * 
48   * @author stack
49   */
50  public class Handler extends URLStreamHandler {
51      protected URLConnection openConnection(URL u)
52      throws IOException {
53          // This looking for accessKey id and accessKey secret code is based
54          // on code from hadoop S3.
55          String accessKey = null;
56          String secretAccessKey = null;
57          String userInfo = u.getUserInfo();
58          if (userInfo != null) {
59              int index = userInfo.indexOf(':');
60              if (index != -1) {
61                accessKey = userInfo.substring(0, index);
62                secretAccessKey = userInfo.substring(index + 1);
63              } else {
64                accessKey = userInfo;
65              }
66          }
67          if (accessKey == null) {
68            accessKey = System.getProperty("aws.access.key.id");
69          }
70          if (secretAccessKey == null) {
71            secretAccessKey = System.getProperty("aws.access.key.secret");
72          }
73          if (accessKey == null && secretAccessKey == null) {
74            throw new IllegalArgumentException("AWS " +
75                  "Access Key ID and Secret Access Key " +
76                  "must be specified as the username " +
77                  "or password (respectively) of a s3 URL, " +
78                  "or by setting the " +
79                  "aws.access.key.id or " +                
80                  "aws.access.key.secret properties (respectively).");
81          } else if (accessKey == null) {
82            throw new IllegalArgumentException("AWS " +
83                  "Access Key ID must be specified " +
84                  "as the username of a s3 URL, or by setting the " +
85                  "aws.access.key.id property.");
86          } else if (secretAccessKey == null) {
87            throw new IllegalArgumentException("AWS " +
88                  "Secret Access Key must be specified " +
89                  "as the password of a s3 URL, or by setting the " +
90                  "aws.access.key.secret property.");        
91          }
92          
93          RestS3Service s3Service;
94          try {
95              s3Service = new RestS3Service(
96                  new AWSCredentials(accessKey, secretAccessKey));
97          } catch (S3ServiceException e) {
98              e.printStackTrace();
99              throw new IOException(e.toString());
100         }
101         InputStream is = null;
102         try {
103             // This opens the stream to the bucket/key object.
104             S3Object s3obj = s3Service.getObject(new S3Bucket(u.getHost()),
105                 u.getPath().substring(1) /* Skip starting '/' character */);
106             is = s3obj.getDataInputStream();
107         } catch (S3ServiceException e) {
108             e.printStackTrace();
109             throw new IOException(e.toString());
110         }
111 
112         final InputStream inputStream = is;
113         return new URLConnection(u) {
114             private InputStream is = inputStream;
115             @Override
116             public InputStream getInputStream() throws IOException {
117                 return this.is;
118             }
119             @Override
120             public void connect() throws IOException {
121                 // Nothing to do. When we give back this object, we're
122                 // connected.
123             }
124         };
125     }
126 
127     /***
128      * Main dumps rsync file to STDOUT.
129      * @param args
130      * @throws IOException
131      */
132     public static void main(String[] args)
133     throws IOException {
134         if (args.length != 1) {
135             System.out.println("Usage: java " +
136                 "org.archive.net.s3.Handler " +
137                 "s3://AWS_ACCESS_KEY_ID:AWS_ACCESS_KEY_SECRET@BUCKET/KEY");
138             System.exit(1);
139         }
140         URL u = new URL(args[0]);
141         URLConnection connect = u.openConnection();
142         // Write download to stdout.
143         final int bufferlength = 4096;
144         byte [] buffer = new byte [bufferlength];
145         InputStream is = connect.getInputStream();
146         try {
147             for (int count = -1;
148                     (count = is.read(buffer, 0, bufferlength)) != -1;) {
149                 System.out.write(buffer, 0, count);
150             }
151             System.out.flush();
152         } finally {
153             is.close();
154         }
155     }
156 }