View Javadoc

    1   /* Copyright (C) 2003 Internet Archive.
    2    *
    3    * This file is part of the Heritrix web crawler (crawler.archive.org).
    4    *
    5    * Heritrix is free software; you can redistribute it and/or modify
    6    * it under the terms of the GNU Lesser Public License as published by
    7    * the Free Software Foundation; either version 2.1 of the License, or
    8    * any later version.
    9    *
    10   * Heritrix is distributed in the hope that it will be useful,
    11   * but WITHOUT ANY WARRANTY; without even the implied warranty of
    12   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    13   * GNU Lesser Public License for more details.
    14   *
    15   * You should have received a copy of the GNU Lesser Public License
    16   * along with Heritrix; if not, write to the Free Software
    17   * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
    18   *
    19   * Created on Jan 15, 2004
    20   *
    21   */
    22  package org.archive.crawler.extractor;
    23
    24  import java.io.IOException;
    25  import java.io.InputStream;
    26  import java.util.regex.Matcher;
    27
    28  import javax.management.AttributeNotFoundException;
    29
    30  import org.apache.commons.io.IOUtils;
    31  import org.archive.crawler.datamodel.CoreAttributeConstants;
    32  import org.archive.crawler.datamodel.CrawlURI;
    33  import org.archive.crawler.settings.SimpleType;
    34  import org.archive.crawler.settings.Type;
    35  import org.archive.net.UURI;
    36  import org.archive.util.TextUtils;
    37
    38  /***
    39   * A last ditch extractor that will look at the raw byte code and try to extract
    40   * anything that <i>looks</i> like a link.
    41   *
    42   * If used, it should always be specified as the last link extractor in the
    43   * order file.
    44   * <p>
    45   * To accomplish this it will scan through the bytecode and try and build up
    46   * strings of consecutive bytes that all represent characters that are valid
    47   * in a URL (see #isURLableChar(int) for details).
    48   * Once it hits the end of such a string (i.e. finds a character that
    49   * should not be in a URL) it will try to determine if it has found a URL.
    50   * This is done be seeing if the string is an IP address prefixed with
    51   * http(s):// or contains a dot followed by a Top Level Domain and end of
    52   * string or a slash.
    53   *
    54   * @author Kristinn Sigurdsson
    55   */
    56  public class ExtractorUniversal extends Extractor
    57  implements CoreAttributeConstants {
    58
    59      private static final long serialVersionUID = -7593380118857156939L;
    60
    61  //    private static final Logger logger =
    62  //        Logger.getLogger(ExtractorUniversal.class.getName());
    63
    64      private static String ATTR_MAX_DEPTH_BYTES = "max-depth-bytes";
    65
    66      /*** Default value for how far into an unknown document we should scan
    67       * - 10k. A value of 0 or lower will disable this.
    68       */
    69      private static long DEFAULT_MAX_DEPTH_BYTES = 10240;
    70
    71      private static String ATTR_MAX_URL_LENGTH = "max-url-length";
    72
    73      /*** Maximum length for a URI that we try to match.*/
    74      private static long DEFAULT_MAX_URL_LENGTH = UURI.MAX_URL_LENGTH;
    75
    76      /***
    77       * Matches any string that begins with http:// or https:// followed by
    78       * something that looks like an ip address (four numbers, none longer then
    79       * 3 chars seperated by 3 dots). Does <b>not</b> ensure that the numbers are
    80       * each in the range 0-255.
    81       */
    82      static final String IP_ADDRESS =
    83          "((http://)|(https://))(//d(//d)?(//d)?//.//d(//d)?(//d)?//.//d(//d)?(//d)?//.//d(//d)?(//d)?)";
    84
    85      /***
    86       * Matches any string that begins with a TLD (no .) followed by a '/' slash
    87       * or end of string. If followed by slash then nothing after the slash is
    88       * of consequence.
    89       */
    90      public static final String TLDs =
    91            "(ac(/.*)?)"  // ac  Ascension Island
    92          + "|(ad(/.*)?)" // ad  Andorra
    93          + "|(ae(/.*)?)" // ae  United Arab Emirates
    94          + "|(af(/.*)?)" // af  Afghanistan
    95          + "|(ag(/.*)?)" // ag  Antigua and Barbuda
    96          + "|(ai(/.*)?)" // ai  Anguilla
    97          + "|(al(/.*)?)" // al  Albania
    98          + "|(am(/.*)?)" // am  Armenia
    99          + "|(an(/.*)?)" // an  Netherlands Antilles
    100         + "|(ao(/.*)?)" // ao  Angola
    101         + "|(aero(/.*)?)" // aero Air-transport industry
    102         + "|(aq(/.*)?)" // aq  Antarctica
    103         + "|(ar(/.*)?)" // ar  Argentina
    104         + "|(as(/.*)?)" // as  American Samoa
    105         + "|(at(/.*)?)" // at  Austria
    106         + "|(au(/.*)?)" // au  Australia
    107         + "|(aw(/.*)?)" // aw  Aruba
    108         + "|(az(/.*)?)" // az  Azerbaijan
    109         + "|(ba(/.*)?)" // ba  Bosnia Hercegovina
    110         + "|(bb(/.*)?)" // bb  Barbados
    111         + "|(bd(/.*)?)" // bd  Bangladesh
    112         + "|(be(/.*)?)" // be  Belgium
    113         + "|(bf(/.*)?)" // bf  Burkina Faso
    114         + "|(bg(/.*)?)" // bg  Bulgaria
    115         + "|(bh(/.*)?)" // bh  Bahrain
    116         + "|(bi(/.*)?)" // bi  Burundi
    117         + "|(biz(/.*)?)" // biz Businesses
    118         + "|(bj(/.*)?)" // bj  Benin
    119         + "|(bm(/.*)?)" // bm  Bermuda
    120         + "|(bn(/.*)?)" // bn  Brunei Darussalam
    121         + "|(bo(/.*)?)" // bo  Bolivia
    122         + "|(br(/.*)?)" // br  Brazil
    123         + "|(bs(/.*)?)" // bs  Bahamas
    124         + "|(bt(/.*)?)" // bt  Bhutan
    125         + "|(bv(/.*)?)" // bv  Bouvet Island
    126         + "|(bw(/.*)?)" // bw  Botswana
    127         + "|(by(/.*)?)" // by  Belarus (Byelorussia)
    128         + "|(bz(/.*)?)" // bz  Belize
    129         + "|(ca(/.*)?)" // ca  Canada
    130         + "|(cc(/.*)?)" // cc  Cocos Islands (Keeling)
    131         + "|(cd(/.*)?)" // cd  Congo, Democratic Republic of the
    132         + "|(cf(/.*)?)" // cf  Central African Republic
    133         + "|(cg(/.*)?)" // cg  Congo, Republic of
    134         + "|(ch(/.*)?)" // ch  Switzerland
    135         + "|(ci(/.*)?)" // ci  Cote d'Ivoire (Ivory Coast)
    136         + "|(ck(/.*)?)" // ck  Cook Islands
    137         + "|(cl(/.*)?)" // cl  Chile
    138         + "|(cm(/.*)?)" // cm  Cameroon
    139         + "|(cn(/.*)?)" // cn  China
    140         + "|(co(/.*)?)" // co  Colombia
    141         + "|(com(/.*)?)" // com Commercial
    142         + "|(coop(/.*)?)" // coop Cooperatives
    143         + "|(cr(/.*)?)" // cr  Costa Rica
    144         + "|(cs(/.*)?)" // cs  Czechoslovakia
    145         + "|(cu(/.*)?)" // cu  Cuba
    146         + "|(cv(/.*)?)" // cv  Cap Verde
    147         + "|(cx(/.*)?)" // cx  Christmas Island
    148         + "|(cy(/.*)?)" // cy  Cyprus
    149         + "|(cz(/.*)?)" // cz  Czech Republic
    150         + "|(de(/.*)?)" // de  Germany
    151         + "|(dj(/.*)?)" // dj  Djibouti
    152         + "|(dk(/.*)?)" // dk  Denmark
    153         + "|(dm(/.*)?)" // dm  Dominica
    154         + "|(do(/.*)?)" // do  Dominican Republic
    155         + "|(dz(/.*)?)" // dz  Algeria
    156         + "|(ec(/.*)?)" // ec  Ecuador
    157         + "|(edu(/.*)?)" // edu Educational Institution
    158         + "|(ee(/.*)?)" // ee  Estonia
    159         + "|(eg(/.*)?)" // eg  Egypt
    160         + "|(eh(/.*)?)" // eh  Western Sahara
    161         + "|(er(/.*)?)" // er  Eritrea
    162         + "|(es(/.*)?)" // es  Spain
    163         + "|(et(/.*)?)" // et  Ethiopia
    164         + "|(fi(/.*)?)" // fi  Finland
    165         + "|(fj(/.*)?)" // fj  Fiji
    166         + "|(fk(/.*)?)" // fk  Falkland Islands
    167         + "|(fm(/.*)?)" // fm  Micronesia, Federal State of
    168         + "|(fo(/.*)?)" // fo  Faroe Islands
    169         + "|(fr(/.*)?)" // fr  France
    170         + "|(ga(/.*)?)" // ga  Gabon
    171         + "|(gd(/.*)?)" // gd  Grenada
    172         + "|(ge(/.*)?)" // ge  Georgia
    173         + "|(gf(/.*)?)" // gf  French Guiana
    174         + "|(gg(/.*)?)" // gg  Guernsey
    175         + "|(gh(/.*)?)" // gh  Ghana
    176         + "|(gi(/.*)?)" // gi  Gibraltar
    177         + "|(gl(/.*)?)" // gl  Greenland
    178         + "|(gm(/.*)?)" // gm  Gambia
    179         + "|(gn(/.*)?)" // gn  Guinea
    180         + "|(gov(/.*)?)" // gov Government (US)
    181         + "|(gp(/.*)?)" // gp  Guadeloupe
    182         + "|(gq(/.*)?)" // gq  Equatorial Guinea
    183         + "|(gr(/.*)?)" // gr  Greece
    184         + "|(gs(/.*)?)" // gs  South Georgia and the South Sandwich Islands
    185         + "|(gt(/.*)?)" // gt  Guatemala
    186         + "|(gu(/.*)?)" // gu  Guam
    187         + "|(gw(/.*)?)" // gw  Guinea-Bissau
    188         + "|(gy(/.*)?)" // gy  Guyana
    189         + "|(hk(/.*)?)" // hk  Hong Kong
    190         + "|(hm(/.*)?)" // hm  Heard and McDonald Islands
    191         + "|(hn(/.*)?)" // hn  Honduras
    192         + "|(hr(/.*)?)" // hr  Croatia/Hrvatska
    193         + "|(ht(/.*)?)" // ht  Haiti
    194         + "|(hu(/.*)?)" // hu  Hungary
    195         + "|(id(/.*)?)" // id  Indonesia
    196         + "|(ie(/.*)?)" // ie  Ireland
    197         + "|(il(/.*)?)" // il  Israel
    198         + "|(im(/.*)?)" // im  Isle of Man
    199         + "|(in(/.*)?)" // in  India
    200         + "|(info(/.*)?)" // info
    201         + "|(int(/.*)?)" // int Int. Organizations
    202         + "|(io(/.*)?)" // io  British Indian Ocean Territory
    203         + "|(iq(/.*)?)" // iq  Iraq
    204         + "|(ir(/.*)?)" // ir  Iran, Islamic Republic of
    205         + "|(is(/.*)?)" // is  Iceland
    206         + "|(it(/.*)?)" // it  Italy
    207         + "|(je(/.*)?)" // je  Jersey
    208         + "|(jm(/.*)?)" // jm  Jamaica
    209         + "|(jo(/.*)?)" // jo  Jordan
    210         + "|(jp(/.*)?)" // jp  Japan
    211         + "|(ke(/.*)?)" // ke  Kenya
    212         + "|(kg(/.*)?)" // kg  Kyrgyzstan
    213         + "|(kh(/.*)?)" // kh  Cambodia
    214         + "|(ki(/.*)?)" // ki  Kiribati
    215         + "|(km(/.*)?)" // km  Comoros
    216         + "|(kn(/.*)?)" // kn  Saint Kitts and Nevis
    217         + "|(kp(/.*)?)" // kp  Korea, Democratic People's Republic
    218         + "|(kr(/.*)?)" // kr  Korea, Republic of
    219         + "|(kw(/.*)?)" // kw  Kuwait
    220         + "|(ky(/.*)?)" // ky  Cayman Islands
    221         + "|(kz(/.*)?)" // kz  Kazakhstan
    222         + "|(la(/.*)?)" // la  Lao People's Democratic Republic
    223         + "|(lb(/.*)?)" // lb  Lebanon
    224         + "|(lc(/.*)?)" // lc  Saint Lucia
    225         + "|(li(/.*)?)" // li  Liechtenstein
    226         + "|(lk(/.*)?)" // lk  Sri Lanka
    227         + "|(lr(/.*)?)" // lr  Liberia
    228         + "|(ls(/.*)?)" // ls  Lesotho
    229         + "|(lt(/.*)?)" // lt  Lithuania
    230         + "|(lu(/.*)?)" // lu  Luxembourg
    231         + "|(lv(/.*)?)" // lv  Latvia
    232         + "|(ly(/.*)?)" // ly  Libyan Arab Jamahiriya
    233         + "|(ma(/.*)?)" // ma  Morocco
    234         + "|(mc(/.*)?)" // mc  Monaco
    235         + "|(md(/.*)?)" // md  Moldova, Republic of
    236         + "|(mg(/.*)?)" // mg  Madagascar
    237         + "|(mh(/.*)?)" // mh  Marshall Islands
    238         + "|(mil(/.*)?)" // mil Military (US Dept of Defense)
    239         + "|(mk(/.*)?)" // mk  Macedonia, Former Yugoslav Republic
    240         + "|(ml(/.*)?)" // ml  Mali
    241         + "|(mm(/.*)?)" // mm  Myanmar
    242         + "|(mn(/.*)?)" // mn  Mongolia
    243         + "|(mo(/.*)?)" // mo  Macau
    244         + "|(mp(/.*)?)" // mp  Northern Mariana Islands
    245         + "|(mq(/.*)?)" // mq  Martinique
    246         + "|(mr(/.*)?)" // mr  Mauritani
    247         + "|(ms(/.*)?)" // ms  Montserrat
    248         + "|(mt(/.*)?)" // mt  Malta
    249         + "|(mu(/.*)?)" // mu  Mauritius
    250         + "|(museum(/.*)?)" // museum Museums
    251         + "|(mv(/.*)?)" // mv  Maldives
    252         + "|(mw(/.*)?)" // mw  Malawi
    253         + "|(mx(/.*)?)" // mx  Mexico
    254         + "|(my(/.*)?)" // my  Malaysia
    255         + "|(mz(/.*)?)" // mz  Mozambique
    256         + "|(na(/.*)?)" // na  Namibia
    257         + "|(name(/.*)?)" // name Individuals
    258         + "|(nc(/.*)?)" // nc  New Caledonia
    259         + "|(ne(/.*)?)" // ne  Niger
    260         + "|(net(/.*)?)" // net networks
    261         + "|(nf(/.*)?)" // nf  Norfolk Island
    262         + "|(ng(/.*)?)" // ng  Nigeria
    263         + "|(ni(/.*)?)" // ni  Nicaragua
    264         + "|(nl(/.*)?)" // nl  Netherlands
    265         + "|(no(/.*)?)" // no  Norway
    266         + "|(np(/.*)?)" // np  Nepal
    267         + "|(nr(/.*)?)" // nr  Nauru
    268         + "|(nt(/.*)?)" // nt  Neutral Zone
    269         + "|(nu(/.*)?)" // nu  Niue
    270         + "|(nz(/.*)?)" // nz  New Zealand
    271         + "|(om(/.*)?)" // om  Oman
    272         + "|(org(/.*)?)" // org Organization (non-profit)
    273         + "|(pa(/.*)?)" // pa  Panama
    274         + "|(pe(/.*)?)" // pe  Peru
    275         + "|(pf(/.*)?)" // pf  French Polynesia
    276         + "|(pg(/.*)?)" // pg  Papua New Guinea
    277         + "|(ph(/.*)?)" // ph  Philippines
    278         + "|(pk(/.*)?)" // pk  Pakistan
    279         + "|(pl(/.*)?)" // pl  Poland
    280         + "|(pm(/.*)?)" // pm  St. Pierre and Miquelon
    281         + "|(pn(/.*)?)" // pn  Pitcairn Island
    282         + "|(pr(/.*)?)" // pr  Puerto Rico
    283         + "|(pro(/.*)?)" // pro Accountants, lawyers, and physicians
    284         + "|(ps(/.*)?)" // ps  Palestinian Territories
    285         + "|(pt(/.*)?)" // pt  Portugal
    286         + "|(pw(/.*)?)" // pw  Palau
    287         + "|(py(/.*)?)" // py  Paraguay
    288         + "|(qa(/.*)?)" // qa  Qatar
    289         + "|(re(/.*)?)" // re  Reunion Island
    290         + "|(ro(/.*)?)" // ro  Romania
    291         + "|(ru(/.*)?)" // ru  Russian Federation
    292         + "|(rw(/.*)?)" // rw  Rwanda
    293         + "|(sa(/.*)?)" // sa  Saudi Arabia
    294         + "|(sb(/.*)?)" // sb  Solomon Islands
    295         + "|(sc(/.*)?)" // sc  Seychelles
    296         + "|(sd(/.*)?)" // sd  Sudan
    297         + "|(se(/.*)?)" // se  Sweden
    298         + "|(sg(/.*)?)" // sg  Singapore
    299         + "|(sh(/.*)?)" // sh  St. Helena
    300         + "|(si(/.*)?)" // si  Slovenia
    301         + "|(sj(/.*)?)" // sj  Svalbard and Jan Mayen Islands
    302         + "|(sk(/.*)?)" // sk  Slovak Republic
    303         + "|(sl(/.*)?)" // sl  Sierra Leone
    304         + "|(sm(/.*)?)" // sm  San Marino
    305         + "|(sn(/.*)?)" // sn  Senegal
    306         + "|(so(/.*)?)" // so  Somalia
    307         + "|(sr(/.*)?)" // sr  Suriname
    308         + "|(sv(/.*)?)" // sv  El Salvador
    309         + "|(st(/.*)?)" // st  Sao Tome and Principe
    310         + "|(sy(/.*)?)" // sy  Syrian Arab Republic
    311         + "|(sz(/.*)?)" // sz  Swaziland
    312         + "|(tc(/.*)?)" // tc  Turks and Caicos Islands
    313         + "|(td(/.*)?)" // td  Chad
    314         + "|(tf(/.*)?)" // tf  French Southern Territories
    315         + "|(tg(/.*)?)" // tg  Togo
    316         + "|(th(/.*)?)" // th  Thailand
    317         + "|(tj(/.*)?)" // tj  Tajikistan
    318         + "|(tk(/.*)?)" // tk  Tokelau
    319         + "|(tm(/.*)?)" // tm  Turkmenistan
    320         + "|(tn(/.*)?)" // tn  Tunisia
    321         + "|(to(/.*)?)" // to  Tonga
    322         + "|(tp(/.*)?)" // tp  East Timor
    323         + "|(tr(/.*)?)" // tr  Turkey
    324         + "|(tt(/.*)?)" // tt  Trinidad and Tobago
    325         + "|(tv(/.*)?)" // tv  Tuvalu
    326         + "|(tw(/.*)?)" // tw  Taiwan
    327         + "|(tz(/.*)?)" // tz  Tanzania
    328         + "|(ua(/.*)?)" // ua  Ukraine
    329         + "|(ug(/.*)?)" // ug  Uganda
    330         + "|(uk(/.*)?)" // uk  United Kingdom
    331         + "|(um(/.*)?)" // um  US Minor Outlying Islands
    332         + "|(us(/.*)?)" // us  United States
    333         + "|(uy(/.*)?)" // uy  Uruguay
    334         + "|(uz(/.*)?)" // uz  Uzbekistan
    335         + "|(va(/.*)?)" // va  Holy See (City Vatican State)
    336         + "|(vc(/.*)?)" // vc  Saint Vincent and the Grenadines
    337         + "|(ve(/.*)?)" // ve  Venezuela
    338         + "|(vg(/.*)?)" // vg  Virgin Islands (British)
    339         + "|(vi(/.*)?)" // vi  Virgin Islands (USA)
    340         + "|(vn(/.*)?)" // vn  Vietnam
    341         + "|(vu(/.*)?)" // vu  Vanuatu
    342         + "|(wf(/.*)?)" // wf  Wallis and Futuna Islands
    343         + "|(ws(/.*)?)" // ws  Western Samoa
    344         + "|(ye(/.*)?)" // ye  Yemen
    345         + "|(yt(/.*)?)" // yt  Mayotte
    346         + "|(yu(/.*)?)" // yu  Yugoslavia
    347         + "|(za(/.*)?)" // za  South Africa
    348         + "|(zm(/.*)?)" // zm  Zambia
    349         + "|(zw(/.*)?)" // zw  Zimbabwe
    350         ;
    351
    352     protected long numberOfCURIsHandled = 0;
    353     protected long numberOfLinksExtracted= 0;
    354
    355     /***
    356      * Constructor
    357      * @param name The name of the module.
    358      */
    359     public ExtractorUniversal(String name) {
    360         super(name, "Link extraction on unknown file types. A best effort" +
    361                 " extractor that looks at the raw byte code of any file " +
    362                 "that has not been handled by another extractor and tries" +
    363                 " to find URIs. Will only match absolute URIs.");
    364         Type e;
    365         e = addElementToDefinition(new SimpleType(ATTR_MAX_DEPTH_BYTES,
    366             "How deep to look into files for URI strings, in bytes",
    367             new Long(DEFAULT_MAX_DEPTH_BYTES)));
    368         e.setExpertSetting(true);
    369         e = addElementToDefinition(new SimpleType(ATTR_MAX_URL_LENGTH,
    370             "Max length of URIs in bytes", new Long(DEFAULT_MAX_URL_LENGTH)));
    371         e.setExpertSetting(true);
    372     }
    373
    374     protected void extract(CrawlURI curi) {
    375         if (!isHttpTransactionContentToProcess(curi)) {
    376             return;
    377         }
    378
    379         numberOfCURIsHandled++;
    380
    381         InputStream instream = null;
    382         try {
    383             instream = curi.getHttpRecorder().getRecordedInput().
    384                 getContentReplayInputStream();
    385             int ch = instream.read();
    386             StringBuffer lookat = new StringBuffer();
    387             long counter = 0;
    388             long maxdepth = ((Long)getAttribute(ATTR_MAX_DEPTH_BYTES,curi)).
    389                 longValue();
    390             if(maxdepth<=0){
    391                 maxdepth = Long.MAX_VALUE;
    392             }
    393             long maxURLLength = ((Long)getAttribute(ATTR_MAX_URL_LENGTH,curi)).
    394                 longValue();
    395             boolean foundDot = false;
    396             while(ch != -1 && ++counter <= maxdepth) {
    397                 if(lookat.length()>maxURLLength){
    398                     //Exceeded maximum length of a URL. Start fresh.
    399                     lookat = new StringBuffer();
    400                     foundDot = false;
    401                 }
    402                 else if(isURLableChar(ch)){
    403                     //Add to buffer.
    404                     if(ch == 46){
    405                         // Current character is a dot '.'
    406                         foundDot = true;
    407                     }
    408                     lookat.append((char)ch);
    409                 } else if(lookat.length() > 3 && foundDot) {
    410                     // It takes a bare mininum of 4 characters to form a URL
    411                     // Since we have at least that many let's try link
    412                     // extraction.
    413                     String newURL = lookat.toString();
    414                     if(looksLikeAnURL(newURL))
    415                     {
    416                         // Looks like we found something.
    417
    418                         // Let's start with a little cleanup as we may have
    419                         // junk in front or at the end.
    420                         if(newURL.toLowerCase().indexOf("http") > 0){
    421                             // Got garbage in front of the protocol. Remove.
    422                             newURL = newURL.substring(newURL.toLowerCase().
    423                                 indexOf("http"));
    424                         }
    425                         while(newURL.substring(newURL.length()-1).equals("."))
    426                         {
    427                             // URLs can't end with a dot. Strip it off.
    428                             newURL = newURL.substring(0,newURL.length()-1);
    429                         }
    430
    431                         // And add the URL to speculative embeds.
    432                         numberOfLinksExtracted++;
    433                         curi.createAndAddLink(newURL,Link.SPECULATIVE_MISC,Link.SPECULATIVE_HOP);
    434                     }
    435                     // Reset lookat for next string.
    436                     lookat = new StringBuffer();
    437                     foundDot = false;
    438                 } else if(lookat.length()>0) {
    439                     // Didn't get enough chars. Reset lookat for next string.
    440                     lookat = new StringBuffer();
    441                     foundDot = false;
    442                 }
    443                 ch = instream.read();
    444             }
    445         } catch(IOException e){
    446             //TODO: Handle this exception.
    447             e.printStackTrace();
    448         } catch (AttributeNotFoundException e) {
    449             // TODO Auto-generated catch block
    450             e.printStackTrace();
    451         } finally {
    452             IOUtils.closeQuietly(instream);
    453         }
    454         // Set flag to indicate that link extraction is completed.
    455         curi.linkExtractorFinished();
    456     }
    457
    458     /***
    459      * This method takes a look at a string and determines if it could be a URL.
    460      * To qualify the string must either begin with "http://" (https would also
    461      * work) followed by something that looks like an IP address or contain
    462      * within the string (possible at the end but not at the beginning) a TLD
    463      * (Top Level Domain) preceded by a dot.
    464      *
    465      * @param lookat The string to examine in an effort to determine if it
    466      * could be a URL
    467      * @return True if the string matches the above criteria for a URL.
    468      */
    469     private boolean looksLikeAnURL(String lookat) {
    470         if(lookat.indexOf("http://")==0 || lookat.indexOf("https://")==0){
    471             //Check if the rest of the string looks like an IP address.
    472             //if so return true. Otherwise continue on.
    473             Matcher ip = TextUtils.getMatcher(IP_ADDRESS, lookat);
    474             boolean testVal = ip.matches();
    475             TextUtils.recycleMatcher(ip);
    476             if(testVal){
    477                 return true;
    478             }
    479         }
    480
    481         int dot = lookat.indexOf(".");
    482         if(dot!=0){//An URL can't start with a .tld.
    483             while(dot != -1 && dot < lookat.length()){
    484                 lookat = lookat.substring(dot+1);
    485                 if (isTLD(lookat.substring(0, lookat.length() <= 6?
    486                     lookat.length(): 6)))
    487                 {
    488                     return true;
    489                 }
    490                 dot = lookat.indexOf(".");
    491             }
    492         }
    493
    494         return false;
    495     }
    496
    497     /***
    498      * Checks if a string is equal to known Top Level Domain. The string may
    499      * contain additional characters <i>after</i> the TLD but not before.
    500      * @param potentialTLD The string (usually 2-6 chars) to check if it starts
    501      * with a TLD.
    502      * @return True if the given string starts with the name of a known TLD
    503      *
    504      * @see #TLDs
    505      */
    506     private boolean isTLD(String potentialTLD) {
    507         if(potentialTLD.length()<2){
    508             return false;
    509         }
    510
    511         potentialTLD.toLowerCase();
    512         Matcher uri = TextUtils.getMatcher(TLDs, potentialTLD);
    513         boolean ret = uri.matches();
    514         TextUtils.recycleMatcher(uri);
    515         return ret;
    516     }
    517
    518     /***
    519      * Determines if a char (as represented by an int in the range of 0-255) is
    520      * a character (in the Ansi character set) that can be present in a URL.
    521      * This method takes a <b>strict</b> approach to what characters can be in
    522      * a URL.
    523      * <p>
    524      * The following are considered to be 'URLable'<br>
    525      * <ul>
    526      *  <li> <code># $ % & + , - . /</code> values 35-38,43-47
    527      *  <li> <code>[0-9]</code> values 48-57
    528      *  <li> <code>: ; = ? @</code> value 58-59,61,63-64
    529      *  <li> <code>[A-Z]</code> values 65-90
    530      *  <li> <code>_</code> value 95
    531      *  <li> <code>[a-z]</code> values 97-122
    532      *  <li> <code>~</code> value 126
    533      * </ul>
    534      * <p>
    535      * To summerize, the following ranges are considered URLable:<br>
    536      * 35-38,43-59,61,63-90,95,97-122,126
    537      *
    538      * @param ch The character (represented by an int) to test.
    539      * @return True if it is a URLable character, false otherwise.
    540      */
    541     private boolean isURLableChar(int ch) {
    542         return (ch>=35 && ch<=38)
    543             || (ch>=43 && ch<=59)
    544             || (ch==61)
    545             || (ch>=63 && ch<=90)
    546             || (ch==95)
    547             || (ch>=97 && ch<=122)
    548             || (ch==126);
    549     }
    550
    551     /* (non-Javadoc)
    552      * @see org.archive.crawler.framework.Processor#report()
    553      */
    554     public String report() {
    555         StringBuffer ret = new StringBuffer();
    556         ret.append("Processor: org.archive.crawler.extractor." +
    557             "ExtractorUniversal\n");
    558         ret.append("  Function:          Link extraction on unknown file" +
    559             " types.\n");
    560         ret.append("  CrawlURIs handled: " + numberOfCURIsHandled + "\n");
    561         ret.append("  Links extracted:   " + numberOfLinksExtracted + "\n\n");
    562
    563         return ret.toString();
    564     }
    565 }