    1   /* ComplexType
    2    *
    3    * $Id: ComplexType.java 5028 2007-03-29 23:21:48Z gojomo $
    4    *
    5    * Created on Dec 17, 2003
    6    *
    7    * Copyright (C) 2004 Internet Archive.
    8    *
    9    * This file is part of the Heritrix web crawler (crawler.archive.org).
    10   *
    11   * Heritrix is free software; you can redistribute it and/or modify
    12   * it under the terms of the GNU Lesser Public License as published by
    13   * the Free Software Foundation; either version 2.1 of the License, or
    14   * any later version.
    15   *
    16   * Heritrix is distributed in the hope that it will be useful,
    17   * but WITHOUT ANY WARRANTY; without even the implied warranty of
    18   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    19   * GNU Lesser Public License for more details.
    20   *
    21   * You should have received a copy of the GNU Lesser Public License
    22   * along with Heritrix; if not, write to the Free Software
    23   * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
    24   */
    25  package org.archive.crawler.settings;
    26
    27  import java.util.ArrayList;
    28  import java.util.HashMap;
    29  import java.util.Iterator;
    30  import java.util.List;
    31  import java.util.Map;
    32  import java.util.Stack;
    33  import java.util.logging.Level;
    34  import java.util.logging.Logger;
    35
    36  import javax.management.Attribute;
    37  import javax.management.AttributeList;
    38  import javax.management.AttributeNotFoundException;
    39  import javax.management.DynamicMBean;
    40  import javax.management.InvalidAttributeValueException;
    41  import javax.management.MBeanAttributeInfo;
    42  import javax.management.MBeanException;
    43  import javax.management.MBeanInfo;
    44  import javax.management.ReflectionException;
    45
    46  import org.apache.commons.httpclient.URIException;
    47  import org.archive.crawler.datamodel.CandidateURI;
    48  import org.archive.crawler.datamodel.CrawlOrder;
    49  import org.archive.crawler.datamodel.CrawlURI;
    50  import org.archive.crawler.settings.Constraint.FailedCheck;
    51  import org.archive.net.UURI;
    52
    53  /*** Superclass of all configurable modules.
    54   *
    55   * This class is in many ways the heart of the settings framework. All modules
    56   * that should be configurable extends this class or one of its subclasses.
    57   *
    58   * All subclasses of this class will automatically conform to the
    59   * JMX DynamicMBean. You could then use the {@link #getMBeanInfo()} method to
    60   * investigate which attributes this module supports and then use the
    61   * {@link #getAttribute(String)} and {@link #setAttribute(Attribute)} methods to
    62   * alter the attributes values.
    63   *
    64   * Because the settings framework supports per domain/host settings there is
    65   * also available context sensitive versions of the DynamicMBean methods.
    66   * If you use the non context sensitive methods, it is the global settings
    67   * that will be altered.
    68   *
    69   * @author John Erik Halse
    70   */
    71  public abstract class ComplexType extends Type implements DynamicMBean {
    72      private static Logger logger =
    73          Logger.getLogger("org.archive.crawler.settings.ComplexType");
    74
    75      private transient SettingsHandler settingsHandler;
    76      private transient ComplexType parent;
    77      private String description;
    78      private String absoluteName;
    79      protected final List<Type> definition = new ArrayList<Type>();
    80      protected final Map<String,Type> definitionMap = new HashMap<String,Type>();
    81      private boolean initialized = false;
    82      private String[] preservedFields = new String[0];
    83
    84      /***
    85       * Private constructor to make sure that no one
    86       * instantiates this class with the empty constructor.
    87       */
    88      private ComplexType() {
    89          super(null, null);
    90      }
    91
    92      /*** Creates a new instance of ComplexType.
    93       *
    94       * @param name the name of the element.
    95       * @param description the description of the element.
    96       */
    97      public ComplexType(String name, String description) {
    98          super(name, null);
    99          this.description = description.intern();
    100     }
    101
    102     protected void setAsOrder(SettingsHandler settingsHandler)
    103     throws InvalidAttributeValueException {
    104         this.settingsHandler = settingsHandler;
    105         this.absoluteName = "";
    106         globalSettings().addTopLevelModule((CrawlOrder) this);
    107         addComplexType(settingsHandler.getSettingsObject(null), this);
    108         this.parent = null;
    109     }
    110
    111     /*** Get the global settings object (aka order).
    112      *
    113      * @return the global settings object.
    114      */
    115     public CrawlerSettings globalSettings() {
    116         if (settingsHandler == null) {
    117             return null;
    118         }
    119         return settingsHandler.getSettingsObject(null);
    120     }
    121
    122     public Type addElement(CrawlerSettings settings, Type type)
    123         throws InvalidAttributeValueException {
    124         getOrCreateDataContainer(settings).addElementType(type);
    125         if (type instanceof ComplexType) {
    126             addComplexType(settings, (ComplexType) type);
    127         }
    128         return type;
    129     }
    130
    131     private ComplexType addComplexType(CrawlerSettings settings,
    132             ComplexType object) throws InvalidAttributeValueException {
    133
    134         if (this.settingsHandler == null) {
    135             throw new IllegalStateException("Can't add ComplexType to 'free' ComplexType");
    136         }
    137         setupVariables(object);
    138         settings.addComplexType(object);
    139         if (!object.initialized) {
    140             Iterator it = object.definition.iterator();
    141             while (it.hasNext()) {
    142                 Type t = (Type) it.next();
    143                 object.addElement(settings, t);
    144             }
    145             object.earlyInitialize(settings);
    146         }
    147         object.initialized = true;
    148
    149         return object;
    150     }
    151
    152     private ComplexType replaceComplexType(CrawlerSettings settings,
    153             ComplexType object) throws InvalidAttributeValueException,
    154             AttributeNotFoundException {
    155         if (this.settingsHandler == null) {
    156             throw new IllegalStateException(
    157                     "Can't add ComplexType to 'free' ComplexType");
    158         }
    159         String[] preservedFields = object.getPreservedFields();
    160
    161         setupVariables(object);
    162
    163         DataContainer oldData = settings.getData(object);
    164         settings.addComplexType(object);
    165         DataContainer newData = settings.getData(object);
    166
    167         if (!object.initialized) {
    168             Iterator it = object.definition.iterator();
    169             while (it.hasNext()) {
    170                 Type t = (Type) it.next();
    171
    172                 // Check if attribute should be copied from old object.
    173                 boolean found = false;
    174                 if (preservedFields.length > 0) {
    175                     for (int i = 0; i < preservedFields.length; i++) {
    176                         if (preservedFields[i].equals(t.getName())) {
    177                             found = true;
    178                             break;
    179                         }
    180                     }
    181                 }
    182                 if (found && oldData.copyAttribute(t.getName(), newData)) {
    183                     if (t instanceof ComplexType) {
    184                         object.setupVariables((ComplexType) t);
    185                     }
    186                 } else {
    187                     object.addElement(settings, t);
    188                 }
    189             }
    190             object.earlyInitialize(settings);
    191         }
    192         object.initialized = true;
    193
    194         return object;
    195     }
    196
    197     /*** Set a list of attribute names that the complex type should attempt to
    198      * preserve if the module is exchanged with an other one.
    199      *
    200      * @param preservedFields array of attributenames to preserve.
    201      */
    202     protected void setPreservedFields(String[] preservedFields) {
    203         this.preservedFields = preservedFields;
    204     }
    205
    206     /*** Get a list of attribute names that the complex type should attempt to
    207      * preserve if the module is exchanged with an other one.
    208      *
    209      * @return an array of attributenames to preserve.
    210      */
    211     protected String[] getPreservedFields() {
    212         return this.preservedFields;
    213     }
    214
    215     /*** Get the active data container for this ComplexType for a specific
    216      * settings object.
    217      *
    218      * If no value has been overridden on the settings object for this
    219      * ComplexType, then it traverses up until it find a DataContainer with
    220      * values for this ComplexType.
    221      *
    222      * This method should probably not be called from user code. It is a helper
    223      * method for the settings framework.
    224      *
    225      * @param context Context from which we get settings.
    226      * @return the active DataContainer.
    227      */
    228     protected DataContainer getDataContainerRecursive(Context context) {
    229         if (context.settings == null) {
    230             return null;
    231         }
    232         DataContainer data = context.settings.getData(this);
    233         if (data == null && context.settings.getParent(context.uri) != null) {
    234             context.settings = context.settings.getParent(context.uri);
    235             data = getDataContainerRecursive(context);
    236         }
    237         return data;
    238     }
    239
    240     /*** Get the active data container for this ComplexType for a specific
    241      * settings object.
    242      *
    243      * If the key has not been overridden on the settings object for this
    244      * ComplexType, then it traverses up until it find a DataContainer with
    245      * the key for this ComplexType.
    246      *
    247      * This method should probably not be called from user code. It is a helper
    248      * method for the settings framework.
    249      *
    250      * @param context the settings object for which the {@link DataContainer}
    251      *                 is active.
    252      * @param key the key to look for.
    253      * @return the active DataContainer.
    254      * @throws AttributeNotFoundException
    255      */
    256     protected DataContainer getDataContainerRecursive(Context context,
    257             String key) throws AttributeNotFoundException {
    258         Context c = new Context(context.settings, context.uri);
    259         DataContainer data = getDataContainerRecursive(c);
    260         while (data != null) {
    261             if (data.containsKey(key)) {
    262                 return data;
    263             }
    264             c.settings = data.getSettings().getParent(c.uri);
    265             data = getDataContainerRecursive(c);
    266         }
    267         throw new AttributeNotFoundException(key);
    268     }
    269
    270     /*** Sets up some variables for a new complex type.
    271      *
    272      * The complex type is set up to be an attribute of
    273      * this complex type.
    274      *
    275      * @param object to be set up.
    276      */
    277     private void setupVariables(ComplexType object) {
    278         object.parent = this;
    279         object.settingsHandler = getSettingsHandler();
    280         object.absoluteName =
    281             (getAbsoluteName() + '/' + object.getName()).intern();
    282     }
    283
    284     public SettingsHandler getSettingsHandler() {
    285         return settingsHandler;
    286     }
    287
    288     /*** Get the absolute name of this ComplexType.
    289      *
    290      * The absolute name is like a file path with the name of the element
    291      * prepended by all the parents names separated by slashes.
    292      * @return Absolute name.
    293      */
    294     public String getAbsoluteName() {
    295         return absoluteName;
    296     }
    297
    298     /***
    299      * Get settings object valid for a URI.
    300      * <p/>
    301      * This method takes an object,
    302      * try to convert it into a {@link CrawlURI} and then tries to get the
    303      * settings object from it. If this fails, then the global settings object
    304      * is returned.
    305      * <p/>
    306      * If the requested attribute is not set on this settings
    307      * object it tries its parent until it gets a settings object where this
    308      * attribute is set is found. If nothing is found, global settings is
    309      * returned.
    310      *
    311      * @param o possible {@link CrawlURI}.
    312      * @param attributeName the attribute that should have a value set on the
    313      *            returned settings object.
    314      * @return the settings object valid for the URI.
    315      */
    316     Context getSettingsFromObject(Object o, String attributeName) {
    317         Context context;
    318         if (o == null) {
    319             context = null;
    320         } else if (o instanceof Context) {
    321             context = (Context) o;
    322         } else if (o instanceof CrawlerSettings) {
    323             context = new Context((CrawlerSettings) o, null);
    324         } else if (o instanceof UURI || o instanceof CandidateURI) {
    325             // Try to get settings for URI that has no references to a
    326             // CrawlServer [SIC - CrawlURI may have CrawlServer -gjm]
    327             context = new Context();
    328             context.uri = (o instanceof CandidateURI)?
    329                 ((CandidateURI) o).getUURI(): (UURI)o;
    330             try {
    331                context.settings = getSettingsHandler().
    332                    getSettings(context.uri.getReferencedHost(),
    333                        context.uri);
    334             }
    335             catch (URIException e1) {
    336                 logger.severe("Failed to get host");
    337             }
    338
    339             if (attributeName != null) {
    340                 try {
    341                     context.settings =
    342                         getDataContainerRecursive(context, attributeName).
    343                             getSettings();
    344                 } catch (AttributeNotFoundException e) {
    345                     // Nothing found, globals will be used
    346                 }
    347             }
    348         } else {
    349             logger.warning("Unknown object type: " +
    350                 o.getClass().getName());
    351             context = null;
    352         }
    353
    354         // if settings could not be resolved use globals.
    355         if (context == null) {
    356             context = new Context(globalSettings(), null);
    357         }
    358         return context;
    359     }
    360
    361     /*** Get settings object valid for a URI.
    362     *
    363     * This method takes an object, try to convert it into a {@link CrawlURI}
    364     * and then tries to get the settings object from it. If this fails, then
    365     * the global settings object is returned.
    366     *
    367     * @param o possible {@link CrawlURI}.
    368     * @return the settings object valid for the URI.
    369     */
    370     Context getSettingsFromObject(Object o) {
    371         return getSettingsFromObject(o, null);
    372     }
    373
    374     /*** Returns true if an element is overridden for this settings object.
    375      *
    376      * @param settings the settings object to investigate.
    377      * @param name the name of the element to check.
    378      * @return true if element is overridden for this settings object, false
    379      *              if not set here or is first defined here.
    380      * @throws AttributeNotFoundException if element doesn't exist.
    381      */
    382     public boolean isOverridden(CrawlerSettings settings, String name)
    383             throws AttributeNotFoundException {
    384         settings = settings == null ? globalSettings() : settings;
    385         DataContainer data = settings.getData(this);
    386         if (data == null || !data.containsKey(name)) {
    387             return false;
    388         }
    389
    390         // Try to find attribute, will throw an exception if not found.
    391         Context context = new Context(settings.getParent(), null);
    392         getDataContainerRecursive(context, name);
    393         return true;
    394     }
    395
    396     /*** Obtain the value of a specific attribute from the crawl order.
    397      *
    398      * If the attribute doesn't exist in the crawl order, the default
    399      * value will be returned.
    400      *
    401      * @param name the name of the attribute to be retrieved.
    402      * @return The value of the attribute retrieved.
    403      * @throws AttributeNotFoundException
    404      * @throws MBeanException
    405      * @throws ReflectionException
    406      */
    407     public Object getAttribute(String name)
    408         throws AttributeNotFoundException, MBeanException, ReflectionException {
    409         return getAttribute(null, name);
    410     }
    411
    412     /*** Obtain the value of a specific attribute that is valid for a
    413      * specific CrawlURI.
    414      *
    415      * This method will try to get the attribute from the host settings
    416      * valid for the CrawlURI. If it is not found it will traverse the
    417      * settings up to the order and as a last resort deliver the default
    418      * value. This is also the case if the CrawlURI is null or if the CrawlURI
    419      * hasn't been assigned a CrawlServer.
    420      *
    421      * @param name the name of the attribute to be retrieved.
    422      * @param uri the CrawlURI that this attribute should be valid for.
    423      * @return The value of the attribute retrieved.
    424      * @see #getAttribute(Object settings, String name)
    425      * @throws AttributeNotFoundException
    426      */
    427     public Object getAttribute(String name, CrawlURI uri)
    428         throws AttributeNotFoundException {
    429         return getAttribute(uri, name);
    430     }
    431
    432     /***
    433      * Obtain the value of a specific attribute that is valid for a specific
    434      * CrawlerSettings object.<p>
    435      *
    436      * This method will first try to get a settings object from the supplied
    437      * context, then try to look up the attribute from this settings object. If
    438      * it is not found it will traverse the settings up to the order and as a
    439      * last resort deliver the default value.
    440      *
    441      * @param context the object to get the settings from.
    442      * @param name the name of the attribute to be retrieved.
    443      * @return The value of the attribute retrieved.
    444      * @see CrawlerSettings
    445      * @throws AttributeNotFoundException
    446      */
    447     public Object getAttribute(Object context, String name)
    448         throws AttributeNotFoundException {
    449         Context ctxt = getSettingsFromObject(context);
    450
    451         // If settings is not set, return the default value
    452         if (ctxt.settings == null) {
    453             try {
    454                 return ((Type) definitionMap.get(name)).getDefaultValue();
    455             } catch (NullPointerException e) {
    456                 throw new AttributeNotFoundException(
    457                         "Could not find attribute: " + name);
    458             }
    459         }
    460
    461         return getDataContainerRecursive(ctxt, name).get(name);
    462     }
    463
    464     /***
    465      * Obtain the value of a specific attribute that is valid for a specific
    466      * CrawlerSettings object.
    467      * <p>
    468      *
    469      * This method will first try to get a settings object from the supplied
    470      * context, then try to look up the attribute from this settings object. If
    471      * it is not found it will traverse the settings up to the order and as a
    472      * last resort deliver the default value.
    473      * <p>
    474      *
    475      * The only difference from the {@link #getAttribute(Object, String)}is
    476      * that this method doesn't throw any checked exceptions. If an undefined
    477      * attribute is requested from a ComplexType, it is concidered a bug and a
    478      * runtime exception is thrown instead.
    479      *
    480      * @param context the object to get the settings from.
    481      * @param name the name of the attribute to be retrieved.
    482      * @return The value of the attribute retrieved.
    483      * @see #getAttribute(Object, String)
    484      * @see CrawlerSettings
    485      * @throws IllegalArgumentException
    486      */
    487     public Object getUncheckedAttribute(Object context, String name) {
    488         try {
    489             return getAttribute(context, name);
    490         } catch (AttributeNotFoundException e) {
    491             throw new IllegalArgumentException("Was passed '" + name +
    492                 "' and got this exception: " + e);
    493         }
    494     }
    495
    496     /*** Obtain the value of a specific attribute that is valid for a
    497      * specific CrawlerSettings object.
    498      *
    499      * This method will try to get the attribute from the supplied host
    500      * settings object. If it is not found it will return <code>null</code>
    501      * and not try to investigate the hierarchy of settings.
    502      *
    503      * @param settings the CrawlerSettings object to search for this attribute.
    504      * @param name the name of the attribute to be retrieved.
    505      * @return The value of the attribute retrieved or null if its not set.
    506      * @see CrawlerSettings
    507      * @throws AttributeNotFoundException is thrown if the attribute doesn't
    508      *         exist.
    509      */
    510     public Object getLocalAttribute(CrawlerSettings settings, String name)
    511             throws AttributeNotFoundException {
    512
    513         settings = settings == null ? globalSettings() : settings;
    514
    515         DataContainer data = settings.getData(this);
    516         if (data != null && data.containsKey(name)) {
    517             // Attribute was found return it.
    518             return data.get(name);
    519         }
    520         // Try to find the attribute, will throw an exception if not found.
    521         Context context = new Context(settings, null);
    522         getDataContainerRecursive(context, name);
    523         return null;
    524     }
    525
    526     /*** Set the value of a specific attribute of the ComplexType.
    527      *
    528      * This method sets the specific attribute for the order file.
    529      *
    530      * @param attribute The identification of the attribute to be set and the
    531      *                  value it is to be set to.
    532      * @throws AttributeNotFoundException is thrown if there is no attribute
    533      *         with this name.
    534      * @throws InvalidAttributeValueException is thrown if the attribute is of
    535      *         wrong type and cannot be converted to the right type.
    536      * @throws MBeanException this is to conform to the MBean specification, but
    537      *         this exception is never thrown, though this might change in the
    538      *         future.
    539      * @throws ReflectionException this is to conform to the MBean specification, but
    540      *         this exception is never thrown, though this might change in the
    541      *         future.
    542      * @see javax.management.DynamicMBean#setAttribute(javax.management.Attribute)
    543      */
    544     public synchronized final void setAttribute(Attribute attribute)
    545         throws
    546             AttributeNotFoundException,
    547             InvalidAttributeValueException,
    548             MBeanException,
    549             ReflectionException {
    550         setAttribute(settingsHandler.getSettingsObject(null), attribute);
    551     }
    552
    553     /*** Set the value of a specific attribute of the ComplexType.
    554      *
    555      * This method is an extension to the Dynamic MBean specification so that
    556      * it is possible to set the value for a CrawlerSettings object other than
    557      * the settings object representing the order.
    558      *
    559      * @param settings the settings object for which this attributes value is valid
    560      * @param attribute The identification of the attribute to be set and the
    561      *                  value it is to be set to.
    562      * @throws AttributeNotFoundException is thrown if there is no attribute
    563      *         with this name.
    564      * @throws InvalidAttributeValueException is thrown if the attribute is of
    565      *         wrong type and cannot be converted to the right type.
    566      * @see javax.management.DynamicMBean#setAttribute(javax.management.Attribute)
    567      */
    568     public synchronized final void setAttribute(CrawlerSettings settings,
    569             Attribute attribute) throws InvalidAttributeValueException,
    570             AttributeNotFoundException {
    571
    572         if(settings==null){
    573             settings = globalSettings();
    574         }
    575
    576         DataContainer data = getOrCreateDataContainer(settings);
    577         Object value = attribute.getValue();
    578
    579         ModuleAttributeInfo attrInfo = (ModuleAttributeInfo) getAttributeInfo(
    580                 settings.getParent(), attribute.getName());
    581
    582         ModuleAttributeInfo localAttrInfo = (ModuleAttributeInfo) data
    583                 .getAttributeInfo(attribute.getName());
    584
    585         // Check if attribute exists
    586         if (attrInfo == null && localAttrInfo == null) {
    587             throw new AttributeNotFoundException(attribute.getName());
    588         }
    589
    590         // Check if we are overriding and if that is allowed for this attribute
    591         if (localAttrInfo == null) {
    592             if (!attrInfo.isOverrideable()) {
    593                 throw new InvalidAttributeValueException(
    594                         "Attribute not overrideable: " + attribute.getName());
    595             }
    596             localAttrInfo = new ModuleAttributeInfo(attrInfo);
    597         }
    598
    599         // Check if value is of correct type. If not, see if it is
    600         // a string and try to turn it into right type
    601         Class typeClass = getDefinition(attribute.getName()).getLegalValueType();
    602         if (!(typeClass.isInstance(value)) && value instanceof String) {
    603             try {
    604                 value = SettingsHandler.StringToType((String) value,
    605                         SettingsHandler.getTypeName(typeClass.getName()));
    606             } catch (ClassCastException e) {
    607                 throw new InvalidAttributeValueException(
    608                         "Unable to decode string '" + value + "' into type '"
    609                                 + typeClass.getName() + "'");
    610             }
    611         }
    612
    613         // Check if the attribute value is legal
    614         FailedCheck error = checkValue(settings, attribute.getName(), value);
    615         if (error != null) {
    616             if (error.getLevel() == Level.SEVERE) {
    617                 throw new InvalidAttributeValueException(error.getMessage());
    618             } else if (error.getLevel() == Level.WARNING) {
    619                 if (!getSettingsHandler().fireValueErrorHandlers(error)) {
    620                     throw new InvalidAttributeValueException(error.getMessage());
    621                 }
    622             } else {
    623                 getSettingsHandler().fireValueErrorHandlers(error);
    624             }
    625         }
    626
    627         // Everything ok, set it
    628         localAttrInfo.setType(value);
    629         Object oldValue = data.put(attribute.getName(), localAttrInfo, value);
    630
    631         // If the attribute is a complex type other than the old value,
    632         // make sure that all sub attributes are correctly set
    633         if (value instanceof ComplexType && value != oldValue) {
    634             ComplexType complex = (ComplexType) value;
    635             replaceComplexType(settings, complex);
    636         }
    637     }
    638
    639     /***
    640      * Get the content type definition for an attribute.
    641      *
    642      * @param attributeName the name of the attribute to get definition for.
    643      * @return the content type definition for the attribute.
    644      */
    645     Type getDefinition(String attributeName) {
    646         return (Type) definitionMap.get(attributeName);
    647     }
    648
    649     /***
    650      * Check an attribute to see if it fulfills all the constraints set on the
    651      * definition of this attribute.
    652      *
    653      * @param settings the CrawlerSettings object for which this check was
    654      *            executed.
    655      * @param attributeName the name of the attribute to check.
    656      * @param value the value to check.
    657      * @return null if everything is ok, otherwise it returns a FailedCheck
    658      *         object with detailed information of what went wrong.
    659      */
    660     public FailedCheck checkValue(CrawlerSettings settings,
    661             String attributeName, Object value) {
    662         return checkValue(settings, attributeName,
    663                 getDefinition(attributeName), value);
    664     }
    665
    666     FailedCheck checkValue(CrawlerSettings settings, String attributeName,
    667             Type definition, Object value) {
    668         FailedCheck res = null;
    669
    670         // Check if value fulfills any constraints
    671         List constraints = definition.getConstraints();
    672         if (constraints != null) {
    673             for (Iterator it = constraints.iterator(); it.hasNext()
    674                     && res == null;) {
    675                 res = ((Constraint) it.next()).check(settings, this,
    676                         definition, value);
    677             }
    678         }
    679
    680         return res;
    681     }
    682
    683     /*** Unset an attribute on a per host level.
    684      *
    685      * This methods removes an override on a per host or per domain level.
    686      *
    687      * @param settings the settings object for which the attribute should be
    688      *        unset.
    689      * @param name the name of the attribute.
    690      * @return The removed attribute or null if nothing was removed.
    691      * @throws AttributeNotFoundException is thrown if the attribute name
    692      *         doesn't exist.
    693      */
    694     public Object unsetAttribute(CrawlerSettings settings, String name)
    695             throws AttributeNotFoundException {
    696
    697         if (settings == globalSettings()) {
    698             throw new IllegalArgumentException(
    699                 "Not allowed to unset attributes in Crawl Order.");
    700         }
    701
    702         DataContainer data = settings.getData(this);
    703         if (data != null && data.containsKey(name)) {
    704             // Remove value
    705             return data.removeElement(name);
    706         }
    707
    708         // Value not found. Check if we should return null or throw an exception
    709         // This method throws an exception if not found.
    710         Context context = new Context(settings, null);
    711         getDataContainerRecursive(context, name);
    712         return null;
    713     }
    714
    715     private DataContainer getOrCreateDataContainer(CrawlerSettings settings)
    716         throws InvalidAttributeValueException {
    717
    718         // Get this ComplexType's data container for the submitted settings
    719         DataContainer data = settings.getData(this);
    720
    721         // If there isn't a container, create one
    722         if (data == null) {
    723             ComplexType parent = getParent();
    724             if (parent == null) {
    725                 settings.addTopLevelModule((ModuleType) this);
    726             } else {
    727                 DataContainer parentData =
    728                     settings.getData(parent);
    729                 if (parentData == null) {
    730                     if (this instanceof ModuleType) {
    731                         settings.addTopLevelModule((ModuleType) this);
    732                     } else {
    733                         settings.addTopLevelModule((ModuleType) parent);
    734                         try {
    735                             parent.setAttribute(settings, this);
    736                         } catch (AttributeNotFoundException e) {
    737                             logger.severe(e.getMessage());
    738                         }
    739                     }
    740                 } else {
    741                     globalSettings().getData(parent).copyAttributeInfo(
    742                         getName(),
    743                         parentData);
    744                 }
    745             }
    746
    747             // Create fresh DataContainer
    748             data = settings.addComplexType(this);
    749         }
    750
    751         // Make sure that the DataContainer references right type
    752         if (data.getComplexType() != this) {
    753             if (this instanceof ModuleType) {
    754                 data = settings.addComplexType(this);
    755             }
    756         }
    757         return data;
    758     }
    759
    760     /* (non-Javadoc)
    761      * @see javax.management.DynamicMBean#getAttributes(java.lang.String[])
    762      */
    763     public AttributeList getAttributes(String[] name) {
    764         return null;
    765     }
    766
    767     /* (non-Javadoc)
    768      * @see javax.management.DynamicMBean#setAttributes(javax.management.AttributeList)
    769      */
    770     public AttributeList setAttributes(AttributeList attributes) {
    771         return null;
    772     }
    773
    774     /* (non-Javadoc)
    775      * @see javax.management.DynamicMBean#invoke(java.lang.String, java.lang.Object[], java.lang.String[])
    776      */
    777     public Object invoke(String arg0, Object[] arg1, String[] arg2)
    778         throws MBeanException, ReflectionException {
    779         throw new ReflectionException(
    780             new NoSuchMethodException("No methods to invoke."));
    781     }
    782
    783     /* (non-Javadoc)
    784      * @see javax.management.DynamicMBean#getMBeanInfo()
    785      */
    786     public MBeanInfo getMBeanInfo() {
    787         return getMBeanInfo(globalSettings());
    788     }
    789
    790     public MBeanInfo getMBeanInfo(Object context) {
    791         MBeanAttributeInfoIterator it = getAttributeInfoIterator(context);
    792         MBeanAttributeInfo[] attributes = new MBeanAttributeInfo[it.size()];
    793         int index = 0;
    794         while(it.hasNext()) {
    795             attributes[index++] = (MBeanAttributeInfo) it.next();
    796         }
    797
    798         MBeanInfo info =
    799             new MBeanInfo(getClass().getName(), getDescription(), attributes,
    800                 null, null, null);
    801         return info;
    802     }
    803
    804     /*** Get the effective Attribute info for an element of this type from
    805      * a settings object.
    806      *
    807      * @param settings the settings object for which the Attribute info is
    808      *        effective.
    809      * @param name the name of the element to get the attribute for.
    810      * @return the attribute info
    811      */
    812     public MBeanAttributeInfo getAttributeInfo(CrawlerSettings settings,
    813             String name) {
    814
    815         MBeanAttributeInfo info = null;
    816
    817         Context context = new Context(settings, null);
    818         DataContainer data = getDataContainerRecursive(context);
    819         while (data != null && info == null) {
    820             info = data.getAttributeInfo(name);
    821             if (info == null) {
    822                 context.settings = data.getSettings().getParent();
    823                 data = getDataContainerRecursive(context);
    824             }
    825         }
    826
    827         return info;
    828     }
    829
    830     /*** Get the Attribute info for an element of this type from the global
    831      * settings.
    832      *
    833      * @param name the name of the element to get the attribute for.
    834      * @return the attribute info
    835      */
    836     public MBeanAttributeInfo getAttributeInfo(String name) {
    837         return getAttributeInfo(globalSettings(), name);
    838     }
    839
    840     /*** Get the description of this type
    841      *
    842      * The description should be suitable for showing in a user interface.
    843      *
    844      * @return this type's description
    845      */
    846     public String getDescription() {
    847         return description;
    848     }
    849
    850     /*** Get the parent of this ComplexType.
    851      *
    852      * @return the parent of this ComplexType.
    853      */
    854     public ComplexType getParent() {
    855         return parent;
    856     }
    857
    858     /*** Set the description of this ComplexType
    859      *
    860      * The description should be suitable for showing in a user interface.
    861      *
    862      * @param string the description to set for this type.
    863      */
    864     public void setDescription(String string) {
    865         description = string;
    866     }
    867
    868     /* (non-Javadoc)
    869      * @see org.archive.crawler.settings.Type#getDefaultValue()
    870      */
    871     public Object getDefaultValue() {
    872         return this;
    873     }
    874
    875     /*** Add a new attribute to the definition of this ComplexType.
    876      *
    877      * This method can only be called before the ComplexType has been
    878      * initialized. This usally means that this method is available for
    879      * constructors of subclasses of this class.
    880      *
    881      * @param type the type to add.
    882      * @return the newly added type.
    883      */
    884     public Type addElementToDefinition(Type type) {
    885         if (isInitialized()) {
    886             throw new IllegalStateException(
    887                     "Elements should only be added to definition in the " +
    888                     "constructor.");
    889         }
    890         if (definitionMap.containsKey(type.getName())) {
    891             definition.remove(definitionMap.remove(type.getName()));
    892         }
    893
    894         definition.add(type);
    895         definitionMap.put(type.getName(), type);
    896         return type;
    897     }
    898
    899     /*** Get an element definition from this complex type.
    900      *
    901      * This method can only be called before the ComplexType has been
    902      * initialized. This usally means that this method is available for
    903      * constructors of subclasses of this class.
    904      *
    905      * @param name name of element to get.
    906      * @return the requested element or null if non existent.
    907      */
    908     public Type getElementFromDefinition(String name) {
    909         if (isInitialized()) {
    910             throw new IllegalStateException(
    911                     "Elements definition can only be accessed in the " +
    912                     "constructor.");
    913         }
    914         return (Type) definitionMap.get(name);
    915     }
    916
    917     /***
    918      * This method can only be called before the ComplexType has been
    919      * initialized. This usually means that this method is available for
    920      * constructors of subclasses of this class.
    921      * @param name Name of element to remove.
    922      * @return Element removed.
    923      */
    924     protected Type removeElementFromDefinition(final String name) {
    925         if (isInitialized()) {
    926             throw new IllegalStateException(
    927                 "Elements definition can only be removed in constructor.");
    928         }
    929         Object removedObj = this.definitionMap.remove(name);
    930         if (removedObj != null) {
    931             this.definition.remove(removedObj);
    932         }
    933         return (Type)removedObj;
    934     }
    935
    936     /*** This method can be overridden in subclasses to do local
    937      * initialisation.
    938      *
    939      * This method is run before the class has been updated with
    940      * information from settings files. That implies that if you
    941      * call getAttribute inside this method you will only get the
    942      * default values.
    943      *
    944      * @param settings the CrawlerSettings object for which this
    945      *        complex type is defined.
    946      */
    947     public void earlyInitialize(CrawlerSettings settings) {
    948     }
    949
    950     /*** Returns true if this ComplexType is initialized.
    951      *
    952      * @return true if this ComplexType is initialized.
    953      */
    954     public boolean isInitialized() {
    955         return initialized;
    956     }
    957
    958     public Object[] getLegalValues() {
    959         return null;
    960     }
    961
    962     /*** Returns this object.
    963      *
    964      * This method is implemented to be able to treat the ComplexType as an
    965      * subclass of {@link javax.management.Attribute}.
    966      *
    967      * @return this object.
    968      * @see javax.management.Attribute#getValue()
    969      */
    970     public Object getValue() {
    971         return this;
    972     }
    973
    974     class Context {
    975         CrawlerSettings settings;
    976         UURI uri;
    977
    978         Context() {
    979             settings = null;
    980             uri = null;
    981         }
    982
    983         Context(CrawlerSettings settings, UURI uri) {
    984             this.settings = settings;
    985             this.uri = uri;
    986         }
    987     }
    988
    989     /*** Get an Iterator over all the attributes in this ComplexType.
    990     *
    991     * @param context the context for which this set of attributes are valid.
    992     * @return an iterator over all the attributes in this map.
    993     */
    994    public Iterator iterator(Object context) {
    995        return new AttributeIterator(context);
    996    }
    997
    998    /*** Get an Iterator over all the MBeanAttributeInfo in this ComplexType.
    999    *
    1000    * @param context the context for which this set of MBeanAttributeInfo are valid.
    1001    * @return an iterator over all the MBeanAttributeInfo in this map.
    1002    */
    1003    public MBeanAttributeInfoIterator getAttributeInfoIterator(Object context) {
    1004        return new MBeanAttributeInfoIterator(context);
    1005    }
    1006
    1007    /***
    1008     * Iterator over all attributes in a ComplexType.
    1009     *
    1010     * @author John Erik Halse
    1011     */
    1012    private class AttributeIterator implements Iterator {
    1013        private Context context;
    1014        private Stack<Iterator<MBeanAttributeInfo>> attributeStack
    1015         = new Stack<Iterator<MBeanAttributeInfo>>();
    1016        private Iterator currentIterator;
    1017
    1018        public AttributeIterator(Object ctxt) {
    1019            this.context = getSettingsFromObject(ctxt);
    1020            Context c = new Context(context.settings, context.uri);
    1021            DataContainer data = getDataContainerRecursive(c);
    1022            while (data != null) {
    1023                this.attributeStack.push(data.getLocalAttributeInfoList().
    1024                    iterator());
    1025                c.settings = data.getSettings().getParent();
    1026                data = getDataContainerRecursive(c);
    1027            }
    1028
    1029            this.currentIterator = (Iterator) this.attributeStack.pop();
    1030        }
    1031
    1032        public boolean hasNext() {
    1033            if (this.currentIterator.hasNext()) {
    1034                return true;
    1035            }
    1036            if (this.attributeStack.isEmpty()) {
    1037                return false;
    1038            }
    1039            this.currentIterator = (Iterator) this.attributeStack.pop();
    1040            return this.currentIterator.hasNext();
    1041        }
    1042
    1043        public Object next() {
    1044            hasNext();
    1045            try {
    1046                MBeanAttributeInfo attInfo = (MBeanAttributeInfo) this.currentIterator.next();
    1047                Object attr = getAttribute(this.context, attInfo.getName());
    1048                if (!(attr instanceof Attribute)) {
    1049                    attr = new Attribute(attInfo.getName(), attr);
    1050                }
    1051                return attr;
    1052            } catch (AttributeNotFoundException e) {
    1053                // This should never happen
    1054                e.printStackTrace();
    1055                return null;
    1056            }
    1057        }
    1058
    1059        public void remove() {
    1060            throw new UnsupportedOperationException();
    1061        }
    1062    }
    1063
    1064    /***
    1065     * Iterator over all MBeanAttributeInfo for this ComplexType
    1066     *
    1067     * @author John Erik Halse
    1068     */
    1069    public class MBeanAttributeInfoIterator implements Iterator {
    1070        private Context context;
    1071        private Stack<Iterator<MBeanAttributeInfo>> attributeStack
    1072         = new Stack<Iterator<MBeanAttributeInfo>>();
    1073        private Iterator currentIterator;
    1074        private int attributeCount = 0;
    1075
    1076        public MBeanAttributeInfoIterator(Object ctxt) {
    1077            this.context = getSettingsFromObject(ctxt);
    1078            //Stack attributeStack = new Stack();
    1079            //
    1080            DataContainer data = getDataContainerRecursive(context);
    1081            while (data != null) {
    1082                attributeStack.push(data.getLocalAttributeInfoList().iterator());
    1083                attributeCount += data.getLocalAttributeInfoList().size();
    1084                context.settings = data.getSettings().getParent();
    1085                data = getDataContainerRecursive(context);
    1086            }
    1087
    1088            this.currentIterator = (Iterator) this.attributeStack.pop();
    1089        }
    1090
    1091        public boolean hasNext() {
    1092             if (this.currentIterator.hasNext()) {
    1093                 return true;
    1094             }
    1095             if (this.attributeStack.isEmpty()) {
    1096                 return false;
    1097             }
    1098             this.currentIterator = (Iterator)this.attributeStack.pop();
    1099             return this.currentIterator.hasNext();
    1100         }
    1101
    1102        public Object next() {
    1103            hasNext();
    1104            MBeanAttributeInfo attInfo = (MBeanAttributeInfo) this.currentIterator.next();
    1105            return attInfo;
    1106        }
    1107
    1108        public void remove() {
    1109            throw new UnsupportedOperationException();
    1110        }
    1111
    1112        public int size() {
    1113            return attributeCount;
    1114        }
    1115    }
    1116
    1117    @Override
    1118    public String toString() {
    1119        // In 1.6, toString goes into infinite loop.  Default implementation is
    1120        // return getName() + '=' + getValue() but this class returns itself
    1121        // for a value on which we do a toString... and around we go.  Short
    1122        // circuit it here.
    1123        return getName() + ": " +
    1124            getClass().getName() + "@" + Integer.toHexString(hashCode());
    1125    }
    1126 }