#!/usr/bin/env python
#
# Copyright 2007 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#




"""Main module for datastore admin receiver.

To use, add this to app.yaml:
  builtins:
  - datastore_admin: on
"""


import logging
import operator
import os
import time

from google.appengine.api import app_identity
from google.appengine.api import datastore_errors
from google.appengine.api import users
from google.appengine.ext import deferred
from google.appengine.ext import webapp
from google.appengine.ext.datastore_admin import backup_handler
from google.appengine.ext.datastore_admin import config
from google.appengine.ext.datastore_admin import copy_handler
from google.appengine.ext.datastore_admin import delete_handler
from google.appengine.ext.datastore_admin import utils
from google.appengine.ext.db import metadata
from google.appengine.ext.db import stats
from google.appengine.ext.webapp import util
from google.appengine.runtime import apiproxy_errors





ENTITY_ACTIONS = {
    'Copy to Another App': copy_handler.ConfirmCopyHandler.Render,
    'Delete Entities': delete_handler.ConfirmDeleteHandler.Render,
    'Backup Entities': backup_handler.ConfirmBackupHandler.Render,
}

BACKUP_ACTIONS = {
    'Delete': backup_handler.ConfirmDeleteBackupHandler.Render,
    'Restore': backup_handler.ConfirmRestoreFromBackupHandler.Render,
    'Info': backup_handler.BackupInformationHandler.Render,
}

PENDING_BACKUP_ACTIONS = {
    'Abort': backup_handler.ConfirmAbortBackupHandler.Render,
    'Info': backup_handler.BackupInformationHandler.Render,
}

GET_ACTIONS = ENTITY_ACTIONS.copy()
GET_ACTIONS.update(BACKUP_ACTIONS)
GET_ACTIONS.update(PENDING_BACKUP_ACTIONS)
GET_ACTIONS.update({'Import Backup Information':
                    backup_handler.ConfirmBackupImportHandler.Render})


MAX_RPCS = 10


def _GetDatastoreStats(kinds_list, use_stats_kinds=False):
  """Retrieves stats for kinds.

  Args:
    kinds_list: List of known kinds.
    use_stats_kinds: If stats are available, kinds_list will be ignored and
      all kinds found in stats will be used instead.

  Returns:
    timestamp: records time that statistics were last updated.
    kind_dict: dictionary of kind objects with the following members:
    - kind_name: the name of this kind.
    - count: number of known entities of this type.
    - total_bytes_str: total bytes for this kind as a string.
    - average_bytes_str: average bytes per entity as a string.
  """
  global_stat = stats.GlobalStat.all().fetch(1)
  if not global_stat:
    return _KindsListToTuple(kinds_list)

  global_ts = global_stat[0].timestamp

  kind_stats = stats.KindStat.all().filter('timestamp =', global_ts).fetch(1000)
  if not kind_stats:
    return _KindsListToTuple(kinds_list)

  results = {}
  for kind_ent in kind_stats:



    if (not kind_ent.kind_name.startswith('__')
        and (use_stats_kinds or kind_ent.kind_name in kinds_list)
        and kind_ent.count > 0):
      results[kind_ent.kind_name] = _PresentatableKindStats(kind_ent)

  utils.CacheStats(results.values())



  for kind_str in kinds_list or []:
    if kind_str not in results:
      results[kind_str] = {'kind_name': kind_str}

  return (global_ts,
          sorted(results.values(), key=lambda x: x['kind_name']))


def _KindsListToTuple(kinds_list):
  """Build default tuple when no datastore statistics are available. """
  return '', [{'kind_name': kind} for kind in sorted(kinds_list)]


def _PresentatableKindStats(kind_ent):
  """Generate dict of presentable values for template."""
  count = kind_ent.count
  entity_bytes = kind_ent.entity_bytes
  total_bytes = kind_ent.bytes
  average_bytes = entity_bytes / count
  return {'kind_name': kind_ent.kind_name,
          'count': utils.FormatThousands(kind_ent.count),
          'entity_bytes_str': utils.GetPrettyBytes(entity_bytes),
          'entity_bytes': entity_bytes,
          'total_bytes_str': utils.GetPrettyBytes(total_bytes),
          'total_bytes': total_bytes,
          'average_bytes_str': utils.GetPrettyBytes(average_bytes),
         }


class RouteByActionHandler(webapp.RequestHandler):
  """Route to the appropriate handler based on the action parameter."""

  def ListActions(self, error=None):
    """Handler for get requests to datastore_admin/confirm_delete."""
    use_stats_kinds = False
    kinds = []
    more_kinds = False
    try:
      kinds, more_kinds = self.GetKinds()
      if not kinds:
        use_stats_kinds = True
        logging.warning('Found no kinds. Using datastore stats instead.')
    except datastore_errors.Error, e:
      logging.exception(e)
      use_stats_kinds = True

    last_stats_update, kind_stats = _GetDatastoreStats(
        kinds, use_stats_kinds=use_stats_kinds)

    template_params = {
        'run_as_a_service': self.request.get('run_as_a_service'),
        'datastore_admin_home': utils.GenerateHomeUrl(None),
        'offer_service': (self.request.get('service') and not
                          self.request.get('run_as_a_service')),
        'kind_stats': kind_stats,
        'more_kinds': more_kinds,
        'last_stats_update': last_stats_update,
        'app_id': self.request.get('app_id'),
        'hosting_app_id': app_identity.get_application_id(),
        'has_namespace': self.request.get('namespace', None) is not None,
        'namespace': self.request.get('namespace'),
        'action_list': sorted(ENTITY_ACTIONS.keys()),
        'backup_action_list': sorted(BACKUP_ACTIONS.keys()),
        'pending_backup_action_list': sorted(PENDING_BACKUP_ACTIONS.keys()),
        'error': error,
        'completed_operations': self.GetOperations(active=False),
        'active_operations': self.GetOperations(active=True),
        'pending_backups': self.GetPendingBackups(),
        'backups': self.GetBackups(),
        'map_reduce_path': config.MAPREDUCE_PATH + '/detail'
    }
    utils.RenderToResponse(self, 'list_actions.html', template_params)

  def RouteAction(self, action_dict):
    action = self.request.get('action')
    if not action:
      self.ListActions(error=self.request.get('error', None))
    elif action not in action_dict:
      error = '%s is not a valid action.' % action
      self.ListActions(error=error)
    else:
      action_dict[action](self)

  def get(self):
    self.RouteAction(GET_ACTIONS)

  def post(self):
    self.RouteAction(GET_ACTIONS)

  def GetKinds(self, all_ns=True, deadline=40):
    """Obtain a list of all kind names from the datastore.

    Args:
      all_ns: If true, list kind names for all namespaces.
              If false, list kind names only for the current namespace.
      deadline: maximum number of seconds to spend getting kinds.

    Returns:
      kinds: an alphabetized list of kinds for the specified namespace(s).
      more_kinds: a boolean indicating whether there may be additional kinds
          not included in 'kinds' (e.g. because the query deadline was reached).
    """
    if all_ns:
      kinds, more_kinds = self.GetKindsForAllNamespaces(deadline)
    else:
      kinds, more_kinds = self.GetKindsForCurrentNamespace(deadline)
    return kinds, more_kinds

  def GetKindsForAllNamespaces(self, deadline):
    """Obtain a list of all kind names from the datastore.

    Pulls kinds from all namespaces. The result is deduped and alphabetized.

    Args:
      deadline: maximum number of seconds to spend getting kinds.

    Returns:
      kinds: an alphabetized list of kinds for the specified namespace(s).
      more_kinds: a boolean indicating whether there may be additional kinds
          not included in 'kinds' (e.g. because the query deadline was reached).
    """
    start = time.time()
    kind_name_set = set()

    def ReadFromKindIters(kind_iter_list):
      """Read kinds from a list of iterators.

      Reads a kind from each iterator in kind_iter_list, adds it to
      kind_name_set, and removes any completed iterators.

      Args:
        kind_iter_list: a list of iterators of kinds.
      """
      completed = []
      for kind_iter in kind_iter_list:
        try:
          kind_name = kind_iter.next().kind_name
          if utils.IsKindNameVisible(kind_name):
            kind_name_set.add(kind_name)
        except StopIteration:
          completed.append(kind_iter)
      for kind_iter in completed:
        kind_iter_list.remove(kind_iter)

    more_kinds = False
    try:
      namespace_iter = metadata.Namespace.all().run(batch_size=1000,
                                                    deadline=deadline)
      kind_iter_list = []
      for ns in namespace_iter:


        remaining = deadline - (time.time() - start)

        if remaining <= 0:
          raise datastore_errors.Timeout
        kind_iter_list.append(metadata.Kind.all(namespace=ns.namespace_name)
                              .run(batch_size=1000, deadline=remaining))
        while len(kind_iter_list) == MAX_RPCS:
          ReadFromKindIters(kind_iter_list)
      while kind_iter_list:
        ReadFromKindIters(kind_iter_list)
    except (datastore_errors.Timeout, apiproxy_errors.DeadlineExceededError):
      more_kinds = True
      logging.warning('Failed to retrieve all kinds within deadline.')
    return sorted(kind_name_set), more_kinds

  def GetKindsForCurrentNamespace(self, deadline):
    """Obtain a list of all kind names from the datastore.

    Pulls kinds from the current namespace only. The result is alphabetized.

    Args:
      deadline: maximum number of seconds to spend getting kinds.

    Returns:
      kinds: an alphabetized list of kinds for the specified namespace(s).
      more_kinds: a boolean indicating whether there may be additional kinds
          not included in 'kinds' (e.g. because the query limit was reached).
    """
    more_kinds = False
    kind_names = []
    try:
      kinds = metadata.Kind.all().order('__key__').run(batch_size=1000,
                                                       deadline=deadline)
      for kind in kinds:
        kind_name = kind.kind_name
        if utils.IsKindNameVisible(kind_name):
          kind_names.append(kind_name)
    except (datastore_errors.Timeout, apiproxy_errors.DeadlineExceededError):
      more_kinds = True
      logging.warning('Failed to retrieve all kinds within deadline.')
    return kind_names, more_kinds

  def GetOperations(self, active=False, limit=100):
    """Obtain a list of operation, ordered by last_updated."""
    query = utils.DatastoreAdminOperation.all()
    if active:
      query.filter('status = ', utils.DatastoreAdminOperation.STATUS_ACTIVE)
    else:
      query.filter('status IN ', [
          utils.DatastoreAdminOperation.STATUS_COMPLETED,
          utils.DatastoreAdminOperation.STATUS_FAILED,
          utils.DatastoreAdminOperation.STATUS_ABORTED])
    operations = query.fetch(max(10000, limit) if limit else 1000)
    operations = sorted(operations, key=operator.attrgetter('last_updated'),
                        reverse=True)
    return operations[:limit]

  def GetBackups(self, limit=100):
    """Obtain a list of backups."""
    query = backup_handler.BackupInformation.all()
    query.filter('complete_time > ', 0)
    backups = query.fetch(max(10000, limit) if limit else 1000)
    backups = sorted(backups, key=operator.attrgetter('complete_time'),
                     reverse=True)
    return backups[:limit]

  def GetPendingBackups(self, limit=100):
    """Obtain a list of pending backups."""
    query = backup_handler.BackupInformation.all()
    query.filter('complete_time = ', None)
    backups = query.fetch(max(10000, limit) if limit else 1000)
    backups = sorted(backups, key=operator.attrgetter('start_time'),
                     reverse=True)
    return backups[:limit]


class StaticResourceHandler(webapp.RequestHandler):
  """Read static files from disk."""






  _BASE_FILE_PATH = os.path.dirname(__file__)

  _RESOURCE_MAP = {
      'static/js/compiled.js': 'text/javascript',
      'static/css/compiled.css': 'text/css',
      'static/img/help.gif': 'image/gif',
      'static/img/tip.png': 'image/png',
      'static/img/icn/icn-warning.gif': 'image/gif',
  }

  def get(self):
    relative_path = self.request.path.split(config.BASE_PATH + '/')[1]
    if relative_path not in self._RESOURCE_MAP:
      self.response.set_status(404)
      self.response.out.write('Resource not found.')
      return

    path = os.path.join(self._BASE_FILE_PATH, relative_path)
    self.response.headers['Cache-Control'] = 'public; max-age=300'
    self.response.headers['Content-Type'] = self._RESOURCE_MAP[relative_path]
    if relative_path == 'static/css/compiled.css':


      self.response.out.write(
          open(path).read().replace('url(/img/', 'url(../img/'))
    else:
      self.response.out.write(open(path).read())


class LoginRequiredHandler(webapp.RequestHandler):
  """Handle federated login identity selector page."""

  def get(self):
    target = self.request.get('continue')
    if not target:
      self.error(400)
      return


    login_url = users.create_login_url(target)
    self.redirect(login_url)


def CreateApplication():
  """Create new WSGIApplication and register all handlers.

  Returns:
    an instance of webapp.WSGIApplication with all mapreduce handlers
    registered.
  """
  return webapp.WSGIApplication(
      backup_handler.handlers_list(config.BASE_PATH) +
      copy_handler.handlers_list(config.BASE_PATH) +
      [(r'%s/%s' % (config.BASE_PATH,
                    delete_handler.ConfirmDeleteHandler.SUFFIX),
        delete_handler.ConfirmDeleteHandler),
       (r'%s/%s' % (config.BASE_PATH, delete_handler.DoDeleteHandler.SUFFIX),
        delete_handler.DoDeleteHandler),
       (r'%s/%s' % (config.BASE_PATH, utils.MapreduceDoneHandler.SUFFIX),
        utils.MapreduceDoneHandler),
       (config.DEFERRED_PATH, deferred.TaskHandler),
       (r'%s/static.*' % config.BASE_PATH, StaticResourceHandler),
       (r'/_ah/login_required', LoginRequiredHandler),
       (r'.*', RouteByActionHandler)])


APP = CreateApplication()


def main():
  util.run_wsgi_app(APP)


if __name__ == '__main__':
  main()
