# twitter/models.py
# Brought to you by We Vote. Be good.
# -*- coding: UTF-8 -*-

# See also WeVoteServer/import_export_twitter/models.py for the code that interfaces with twitter (or other) servers

from django.db import models
from import_export_twitter.functions import retrieve_twitter_user_info
from wevote_functions.functions import positive_value_exists

class TwitterUser(models.Model):
    """
    We cache the Twitter info for one handle here. NOTE: multiple accounts can be signed into same Twitter account
    """
    twitter_id = models.BigIntegerField(verbose_name="twitter big integer id", null=True, blank=True)
    twitter_handle = models.CharField(verbose_name='twitter screen name / handle',
                                      max_length=255, null=False, unique=True)
    twitter_name = models.CharField(
        verbose_name="display name from twitter", max_length=255, null=True, blank=True)
    twitter_url = models.URLField(blank=True, null=True, verbose_name='url of user\'s website')

    twitter_profile_image_url_https = models.URLField(verbose_name='url of logo from twitter', blank=True, null=True)
    twitter_location = models.CharField(
        verbose_name="location from twitter", max_length=255, null=True, blank=True)
    twitter_followers_count = models.IntegerField(verbose_name="number of twitter followers",
                                                  null=False, blank=True, default=0)
    twitter_profile_background_image_url_https = models.URLField(verbose_name='tile-able background from twitter',
                                                                 blank=True, null=True)
    twitter_profile_banner_url_https = models.URLField(verbose_name='profile banner image from twitter',
                                                       blank=True, null=True)
    twitter_description = models.CharField(verbose_name="Text description of this organization from twitter.",
                                           max_length=255, null=True, blank=True)

class TwitterUserManager(models.Model):

    def __unicode__(self):
        return "TwitterUserManager"

    def retrieve_twitter_user_locally_or_remotely(self, twitter_handle):
        twitter_user_found = False
        twitter_user = TwitterUser()
        success = False
        status = "TWITTER_USER_NOT_FOUND"

        # Is this twitter_handle already stored locally? If so, return that
        twitter_results = self.retrieve_twitter_user(twitter_handle)
        if twitter_results['twitter_user_found']:
            return twitter_results

        # If here, we want to reach out to Twitter to get info for this twitter_handle
        twitter_results = retrieve_twitter_user_info(twitter_handle)
        if twitter_results['twitter_handle_found']:
            twitter_save_results = self.save_new_twitter_user_from_twitter_json(twitter_results['twitter_json'])
            if twitter_save_results['twitter_user_found']:
                # If saved, pull the fresh results from the database and return
                twitter_second_results = self.retrieve_twitter_user(twitter_handle)
                if twitter_second_results['twitter_user_found']:
                    return twitter_second_results

        results = {
            'success':                  success,
            'status':                   status,
            'twitter_user_found':       twitter_user_found,
            'twitter_user':             twitter_user,
        }
        return results

    def retrieve_twitter_user(self, twitter_handle):
        twitter_user_on_stage = TwitterUser()
        twitter_user_found = False
        success = False

        try:
            if positive_value_exists(twitter_handle):
                status = "RETRIEVE_TWITTER_USER_FOUND_WITH_HANDLE"
                twitter_user_on_stage = TwitterUser.objects.get(twitter_handle__iexact=twitter_handle)
                twitter_user_found = True
                success = True
            else:
                status = "RETRIEVE_TWITTER_USER_INSUFFICIENT_VARIABLES"
        except TwitterUser.MultipleObjectsReturned as e:
            success = False
            status = "RETRIEVE_TWITTER_USER_MULTIPLE_FOUND"
        except TwitterUser.DoesNotExist:
            success = True
            status = "RETRIEVE_TWITTER_USER_NONE_FOUND"

        results = {
            'success':                  success,
            'status':                   status,
            'twitter_user_found':       twitter_user_found,
            'twitter_user':             twitter_user_on_stage,
        }
        return results

    def save_new_twitter_user_from_twitter_json(self, twitter_json):

        if 'screen_name' not in twitter_json:
            results = {
                'success':              False,
                'status':               "SAVE_NEW_TWITTER_USER_MISSING_HANDLE",
                'twitter_user_found':   False,
                'twitter_user':         TwitterUser(),
            }
            return results

        try:
            # Create new twitter_user entry
            twitter_description = twitter_json['description'] if 'description' in twitter_json else ""
            twitter_followers_count = twitter_json['followers_count'] if 'followers_count' in twitter_json else ""
            twitter_handle = twitter_json['screen_name'] if 'screen_name' in twitter_json else ""
            twitter_id = twitter_json['id'] if 'id' in twitter_json else ""
            twitter_location = twitter_json['location'] if 'location' in twitter_json else ""
            twitter_name = twitter_json['name'] if 'name' in twitter_json else ""
            twitter_profile_background_image_url_https = twitter_json['profile_background_image_url_https'] \
                if 'profile_background_image_url_https' in twitter_json else ""
            twitter_profile_banner_url_https = twitter_json['profile_banner_url_https'] \
                if 'profile_banner_url_https' in twitter_json else ""
            twitter_profile_image_url_https = twitter_json['profile_image_url_https'] \
                if 'profile_image_url_https' in twitter_json else ""
            twitter_url = twitter_json['url'] if 'url' in twitter_json else ""

            twitter_user_on_stage = TwitterUser(
                twitter_description=twitter_description,
                twitter_followers_count=twitter_followers_count,
                twitter_handle=twitter_handle,
                twitter_id=twitter_id,
                twitter_location=twitter_location,
                twitter_name=twitter_name,
                twitter_profile_background_image_url_https=twitter_profile_background_image_url_https,
                twitter_profile_banner_url_https=twitter_profile_banner_url_https,
                twitter_profile_image_url_https=twitter_profile_image_url_https,
                twitter_url=twitter_url,
            )
            twitter_user_on_stage.save()
            success = True
            twitter_user_found = True
            status = 'CREATED_TWITTER_USER'
        except Exception as e:
            success = False
            twitter_user_found = False
            status = 'FAILED_TO_CREATE_NEW_TWITTER_USER'
            twitter_user_on_stage = TwitterUser()

        results = {
            'success':                  success,
            'status':                   status,
            'twitter_user_found':       twitter_user_found,
            'twitter_user':             twitter_user_on_stage,
        }
        return results

    def update_or_create_twitter_user(self, twitter_user_id, twitter_user_we_vote_id,
                                    ballot_item_display_name,
                                    contest_office_we_vote_id,
                                    candidate_campaign_we_vote_id,
                                    politician_we_vote_id,
                                    contest_measure_we_vote_id,
                                    info_html,
                                    info_text,
                                    language,
                                    last_editor_we_vote_id,
                                    twitter_user_master_we_vote_id,
                                    more_info_url,
                                    more_info_credit,
                                    google_civic_election_id
                                    ):
        # Does a twitter_user entry already exist?
        twitter_user_manager = TwitterUserManager()
        results = twitter_user_manager.retrieve_twitter_user(twitter_user_id, twitter_user_we_vote_id,
                                                         contest_office_we_vote_id,
                                                         candidate_campaign_we_vote_id,
                                                         politician_we_vote_id,
                                                         contest_measure_we_vote_id)

        twitter_user_on_stage_found = False
        twitter_user_on_stage_id = 0
        twitter_user_on_stage = TwitterUser()
        if results['twitter_user_found']:
            twitter_user_on_stage = results['twitter_user']

            # Update this twitter_user entry with new values - we do not delete because we might be able to use
            # noinspection PyBroadException
            try:
                # Figure out if the update is a change to a master entry
                if positive_value_exists(twitter_user_master_we_vote_id):
                    uses_master_entry = True
                elif (info_html is not False) or (info_text is not False) or (more_info_url is not False):
                    uses_master_entry = False
                elif positive_value_exists(twitter_user_on_stage.info_textx) or \
                        positive_value_exists(twitter_user_on_stage.info_html) or \
                        positive_value_exists(twitter_user_on_stage.more_info_url):
                    uses_master_entry = False
                elif positive_value_exists(twitter_user_on_stage.twitter_user_master_we_vote_id):
                    uses_master_entry = True
                else:
                    uses_master_entry = True

                if ballot_item_display_name is not False:
                    twitter_user_on_stage.ballot_item_display_name = ballot_item_display_name
                if language is not False:
                    twitter_user_on_stage.language = language
                if last_editor_we_vote_id is not False:
                    twitter_user_on_stage.last_editor_we_vote_id = last_editor_we_vote_id
                if contest_office_we_vote_id is not False:
                    twitter_user_on_stage.contest_office_we_vote_id = contest_office_we_vote_id
                if candidate_campaign_we_vote_id is not False:
                    twitter_user_on_stage.candidate_campaign_we_vote_id = candidate_campaign_we_vote_id
                if politician_we_vote_id is not False:
                    twitter_user_on_stage.politician_we_vote_id = politician_we_vote_id
                if contest_measure_we_vote_id is not False:
                    twitter_user_on_stage.contest_measure_we_vote_id = contest_measure_we_vote_id
                if google_civic_election_id is not False:
                    twitter_user_on_stage.google_civic_election_id = google_civic_election_id
                if uses_master_entry:
                    if twitter_user_master_we_vote_id is not False:
                        twitter_user_on_stage.twitter_user_master_we_vote_id = twitter_user_master_we_vote_id
                    # Clear out unique entry values
                    twitter_user_on_stage.info_text = ""
                    twitter_user_on_stage.info_html = ""
                    twitter_user_on_stage.more_info_url = ""
                    twitter_user_on_stage.more_info_credit = NOT_SPECIFIED
                else:
                    # If here, this is NOT a master entry
                    if info_text is not False:
                        twitter_user_on_stage.info_text = info_text
                    if info_html is not False:
                        twitter_user_on_stage.info_html = info_html
                    if more_info_url is not False:
                        twitter_user_on_stage.more_info_url = more_info_url
                    if more_info_credit is not False:
                        twitter_user_on_stage.more_info_credit = more_info_credit
                    # Clear out master entry value
                    twitter_user_on_stage.twitter_user_master_we_vote_id = ""
                if google_civic_election_id is not False:
                    twitter_user_on_stage.google_civic_election_id = google_civic_election_id
                # We don't need to update date_last_changed here because set set auto_now=True in the field
                twitter_user_on_stage.save()
                twitter_user_on_stage_id = twitter_user_on_stage.id
                twitter_user_on_stage_found = True
                status = 'TWITTER_USER_UPDATED'
            except Exception as e:
                status = 'FAILED_TO_UPDATE_TWITTER_USER'
        elif results['MultipleObjectsReturned']:
            status = 'TWITTER_USER MultipleObjectsReturned'
        elif results['DoesNotExist']:
            try:
                # Create new twitter_user entry
                if ballot_item_display_name is False:
                    ballot_item_display_name = ""
                if language is False:
                    language = ENGLISH
                if last_editor_we_vote_id is False:
                    last_editor_we_vote_id = ""
                if contest_office_we_vote_id is False:
                    contest_office_we_vote_id = ""
                if candidate_campaign_we_vote_id is False:
                    candidate_campaign_we_vote_id = ""
                if politician_we_vote_id is False:
                    politician_we_vote_id = ""
                if contest_measure_we_vote_id is False:
                    contest_measure_we_vote_id = ""
                if google_civic_election_id is False:
                    google_civic_election_id = 0
                # Master related data
                if twitter_user_master_we_vote_id is False:
                    twitter_user_master_we_vote_id = ""
                # Unique related data
                if info_html is False:
                    info_html = ""
                if info_text is False:
                    info_text = ""
                if more_info_url is False:
                    more_info_url = ""
                if more_info_credit is False:
                    more_info_credit = None
                twitter_user_on_stage = TwitterUser(
                    ballot_item_display_name=ballot_item_display_name,
                    contest_office_we_vote_id=contest_office_we_vote_id,
                    candidate_campaign_we_vote_id=candidate_campaign_we_vote_id,
                    politician_we_vote_id=politician_we_vote_id,
                    contest_measure_we_vote_id=contest_measure_we_vote_id,
                    info_html=info_html,
                    info_text=info_text,
                    language=language,
                    last_editor_we_vote_id=last_editor_we_vote_id,
                    twitter_user_master_we_vote_id=twitter_user_master_we_vote_id,
                    more_info_url=more_info_url,
                    more_info_credit=more_info_credit,
                    google_civic_election_id=google_civic_election_id
                    # We don't need to update last_updated here because set set auto_now=True in the field
                )
                twitter_user_on_stage.save()
                twitter_user_on_stage_id = twitter_user_on_stage.id
                twitter_user_on_stage_found = True
                status = 'CREATED_TWITTER_USER'
            except Exception as e:
                status = 'FAILED_TO_CREATE_NEW_TWITTER_USER'
                handle_record_not_saved_exception(e, logger=logger, exception_message_optional=status)
        else:
            status = results['status']

        results = {
            'success':            True if twitter_user_on_stage_found else False,
            'status':             status,
            'twitter_user_found':   twitter_user_on_stage_found,
            'twitter_user_id':      twitter_user_on_stage_id,
            'twitter_user':         twitter_user_on_stage,
        }
        return results

    def delete_twitter_user(self, twitter_user_id):
        twitter_user_id = convert_to_int(twitter_user_id)
        twitter_user_deleted = False

        try:
            if twitter_user_id:
                results = self.retrieve_twitter_user(twitter_user_id)
                if results['twitter_user_found']:
                    twitter_user = results['twitter_user']
                    twitter_user_id = twitter_user.id
                    twitter_user.delete()
                    twitter_user_deleted = True
        except Exception as e:
            handle_exception(e, logger=logger)

        results = {
            'success':            twitter_user_deleted,
            'twitter_user_deleted': twitter_user_deleted,
            'twitter_user_id':      twitter_user_id,
        }
        return results

class Tweet(models.Model):
    """
    A tweet referenced somewhere by a We Vote tag. We store it (once - not every time it is referenced by a tag)
    locally so we can publish JSON from for consumption on the We Vote newsfeed.
    """
    # twitter_tweet_id # (unique id from twitter for tweet?)
    author_handle = models.CharField(max_length=15, verbose_name='twitter handle of this tweet\'s author')
    # (stored quickly before we look up voter_id)
    # author_voter_id = models.ForeignKey(Voter, null=True, blank=True, related_name='we vote id of tweet author')
    is_retweet = models.BooleanField(default=False, verbose_name='is this a retweet?')
    # parent_tweet_id # If this is a retweet, what is the id of the originating tweet?
    body = models.CharField(blank=True, null=True, max_length=255, verbose_name='')
    date_published = models.DateTimeField(null=True, verbose_name='date published')

class TweetFavorite(models.Model):
    """
    This table tells us who favorited a tweet
    """
    tweet_id = models.ForeignKey(Tweet, null=True, blank=True, verbose_name='we vote tweet id')
    # twitter_tweet_id # (unique id from twitter for tweet?)
    # TODO Should favorited_by_handle be a ForeignKey link to the Twitter User? I'm concerned this will slow saving,
    #  and it might be better to ForeignKey against voter_id
    favorited_by_handle = models.CharField(
        max_length=15, verbose_name='twitter handle of person who favorited this tweet')
    # (stored quickly before we look up voter_id)
    # favorited_by_voter_id = models.ForeignKey(
    # Voter, null=True, blank=True, related_name='tweet favorited by voter_id')
    date_favorited = models.DateTimeField(null=True, verbose_name='date favorited')

# This should be the master table
class TwitterWhoIFollow(models.Model):
    """
    Other Twitter handles that I follow, from the perspective of handle_of_me
    """
    handle_of_me = models.CharField(max_length=15, verbose_name='from this twitter handle\'s perspective...')
    handle_i_follow = models.CharField(max_length=15, verbose_name='twitter handle being followed')

# This is a we vote copy (for speed) of Twitter handles that follow me. We should have self-healing scripts that set up
#  entries in TwitterWhoIFollow for everyone following someone in the We Vote network, so this table could be flushed
#  and rebuilt at any time
class TwitterWhoFollowMe(models.Model):
    handle_of_me = models.CharField(max_length=15, verbose_name='from this twitter handle\'s perspective...')
    handle_that_follows_me = models.CharField(max_length=15, verbose_name='twitter handle of this tweet\'s author')


import json
import logging
import re
from json import dumps

from six import string_types

from galaxy import model
from galaxy.exceptions import ObjectInvalid
from galaxy.model import LibraryDatasetDatasetAssociation
from galaxy.tools.parameters import update_param
from galaxy.tools.parameters.basic import DataCollectionToolParameter, DataToolParameter, RuntimeValue
from galaxy.tools.parameters.wrapped import WrappedParameters
from galaxy.util import ExecutionTimer
from galaxy.util.none_like import NoneDataset
from galaxy.util.odict import odict
from galaxy.util.template import fill_template
from galaxy.web import url_for

log = logging.getLogger( __name__ )

class ToolExecutionCache( object ):
    """ An object mean to cache calculation caused by repeatedly evaluting
    the same tool by the same user with slightly different parameters.
    """
    def __init__(self, trans):
        self.trans = trans
        self.current_user_roles = trans.get_current_user_roles()

class ToolAction( object ):
    """
    The actions to be taken when a tool is run (after parameters have
    been converted and validated).
    """
    def execute( self, tool, trans, incoming={}, set_output_hid=True ):
        raise TypeError("Abstract method")

class DefaultToolAction( object ):
    """Default tool action is to run an external command"""

    def collect_input_datasets( self, tool, param_values, trans, current_user_roles=None ):
        """
        Collect any dataset inputs from incoming. Returns a mapping from
        parameter name to Dataset instance for each tool parameter that is
        of the DataToolParameter type.
        """
        if current_user_roles is None:
            current_user_roles = trans.get_current_user_roles()
        input_datasets = odict()

        def visitor( input, value, prefix, parent=None, **kwargs ):

            def process_dataset( data, formats=None ):
                if not data or isinstance( data, RuntimeValue ):
                    return None
                if formats is None:
                    formats = input.formats
                if not data.datatype.matches_any( formats ):
                    # Need to refresh in case this conversion just took place, i.e. input above in tool performed the same conversion
                    trans.sa_session.refresh( data )
                    target_ext, converted_dataset = data.find_conversion_destination( formats )
                    if target_ext:
                        if converted_dataset:
                            data = converted_dataset
                        else:
                            data = data.get_converted_dataset( trans, target_ext, target_context=parent )

                if not trans.app.security_agent.can_access_dataset( current_user_roles, data.dataset ):
                    raise Exception( "User does not have permission to use a dataset (%s) provided for input." % data.id )
                return data
            if isinstance( input, DataToolParameter ):
                if isinstance( value, list ):
                    # If there are multiple inputs with the same name, they
                    # are stored as name1, name2, ...
                    for i, v in enumerate( value ):
                        processed_dataset = process_dataset( v )
                        if i == 0:
                            # Allow copying metadata to output, first item will be source.
                            input_datasets[ prefix + input.name ] = processed_dataset
                        input_datasets[ prefix + input.name + str( i + 1 ) ] = processed_dataset
                        conversions = []
                        for conversion_name, conversion_extensions, conversion_datatypes in input.conversions:
                            new_data = process_dataset( input_datasets[ prefix + input.name + str( i + 1 ) ], conversion_datatypes )
                            if not new_data or new_data.datatype.matches_any( conversion_datatypes ):
                                input_datasets[ prefix + conversion_name + str( i + 1 ) ] = new_data
                                conversions.append( ( conversion_name, new_data ) )
                            else:
                                raise Exception('A path for explicit datatype conversion has not been found: %s --/--> %s' % ( input_datasets[ prefix + input.name + str( i + 1 ) ].extension, conversion_extensions ) )
                        if parent:
                            parent[ input.name ][ i ] = input_datasets[ prefix + input.name + str( i + 1 ) ]
                            for conversion_name, conversion_data in conversions:
                                # allow explicit conversion to be stored in job_parameter table
                                parent[ conversion_name ][ i ] = conversion_data.id  # a more robust way to determine JSONable value is desired
                        else:
                            param_values[ input.name ][ i ] = input_datasets[ prefix + input.name + str( i + 1 ) ]
                            for conversion_name, conversion_data in conversions:
                                # allow explicit conversion to be stored in job_parameter table
                                param_values[ conversion_name ][i] = conversion_data.id  # a more robust way to determine JSONable value is desired
                else:
                    input_datasets[ prefix + input.name ] = process_dataset( value )
                    conversions = []
                    for conversion_name, conversion_extensions, conversion_datatypes in input.conversions:
                        new_data = process_dataset( input_datasets[ prefix + input.name ], conversion_datatypes )
                        if not new_data or new_data.datatype.matches_any( conversion_datatypes ):
                            input_datasets[ prefix + conversion_name ] = new_data
                            conversions.append( ( conversion_name, new_data ) )
                        else:
                            raise Exception( 'A path for explicit datatype conversion has not been found: %s --/--> %s' % ( input_datasets[ prefix + input.name ].extension, conversion_extensions ) )
                    target_dict = parent
                    if not target_dict:
                        target_dict = param_values
                    target_dict[ input.name ] = input_datasets[ prefix + input.name ]
                    for conversion_name, conversion_data in conversions:
                        # allow explicit conversion to be stored in job_parameter table
                        target_dict[ conversion_name ] = conversion_data.id  # a more robust way to determine JSONable value is desired
            elif isinstance( input, DataCollectionToolParameter ):
                if not value:
                    return

                dataset_instances = []
                if hasattr( value, 'child_collection' ):
                    # if we are mapping a collection over a tool, we only require the child_collection
                    dataset_instances = value.child_collection.dataset_instances
                else:
                    # else the tool takes a collection as input so we need everything
                    dataset_instances = value.collection.dataset_instances

                for i, v in enumerate( dataset_instances ):
                    data = v
                    if not trans.app.security_agent.can_access_dataset( current_user_roles, data.dataset ):
                        raise Exception( "User does not have permission to use a dataset (%s) provided for input." % data.id )
                    # Skipping implicit conversion stuff for now, revisit at
                    # some point and figure out if implicitly converting a
                    # dataset collection makes senese.
                    input_datasets[ prefix + input.name + str( i + 1 ) ] = data

        tool.visit_inputs( param_values, visitor )
        return input_datasets

    def collect_input_dataset_collections( self, tool, param_values ):
        def append_to_key( the_dict, key, value ):
            if key not in the_dict:
                the_dict[ key ] = []
            the_dict[ key ].append( value )

        input_dataset_collections = dict()

        def visitor( input, value, prefix, parent=None, **kwargs ):
            if isinstance( input, DataToolParameter ):
                values = value
                if not isinstance( values, list ):
                    values = [ value ]
                for i, value in enumerate(values):
                    if isinstance( value, model.HistoryDatasetCollectionAssociation ):
                        append_to_key( input_dataset_collections, prefix + input.name, ( value, True ) )
                        target_dict = parent
                        if not target_dict:
                            target_dict = param_values
                        # This is just a DataToolParameter, so replace this
                        # collection with individual datasets. Database will still
                        # record collection which should be enought for workflow
                        # extraction and tool rerun.
                        dataset_instances = value.collection.dataset_instances
                        if i == 0:
                            target_dict[ input.name ] = []
                        target_dict[ input.name ].extend( dataset_instances )
            elif isinstance( input, DataCollectionToolParameter ):
                append_to_key( input_dataset_collections, prefix + input.name, ( value, False ) )

        tool.visit_inputs( param_values, visitor )
        return input_dataset_collections

    def _check_access( self, tool, trans ):
        assert tool.allow_user_access( trans.user ), "User (%s) is not allowed to access this tool." % ( trans.user )

    def _collect_inputs( self, tool, trans, incoming, history, current_user_roles ):
        """ Collect history as well as input datasets and collections. """
        app = trans.app
        # Set history.
        if not history:
            history = tool.get_default_history_by_trans( trans, create=True )
        if history not in trans.sa_session:
            history = trans.sa_session.query( app.model.History ).get( history.id )

        # Track input dataset collections - but replace with simply lists so collect
        # input datasets can process these normally.
        inp_dataset_collections = self.collect_input_dataset_collections( tool, incoming )
        # Collect any input datasets from the incoming parameters
        inp_data = self.collect_input_datasets( tool, incoming, trans, current_user_roles=current_user_roles )

        return history, inp_data, inp_dataset_collections

    def execute(self, tool, trans, incoming={}, return_job=False, set_output_hid=True, history=None, job_params=None, rerun_remap_job_id=None, mapping_over_collection=False, execution_cache=None ):
        """
        Executes a tool, creating job and tool outputs, associating them, and
        submitting the job to the job queue. If history is not specified, use
        trans.history as destination for tool's output datasets.
        """
        self._check_access( tool, trans )
        app = trans.app
        if execution_cache is None:
            execution_cache = ToolExecutionCache(trans)
        current_user_roles = execution_cache.current_user_roles
        history, inp_data, inp_dataset_collections = self._collect_inputs(tool, trans, incoming, history, current_user_roles)

        # Build name for output datasets based on tool name and input names
        on_text = self._get_on_text( inp_data )

        # format='input" previously would give you a random extension from
        # the input extensions, now it should just give "input" as the output
        # format.
        input_ext = 'data' if tool.profile < 16.04 else "input"
        input_dbkey = incoming.get( "dbkey", "?" )
        for name, data in reversed(inp_data.items()):
            if not data:
                data = NoneDataset( datatypes_registry=app.datatypes_registry )
                continue

            # Convert LDDA to an HDA.
            if isinstance(data, LibraryDatasetDatasetAssociation):
                data = data.to_history_dataset_association( None )
                inp_data[name] = data

            if tool.profile < 16.04:
                input_ext = data.ext

            if data.dbkey not in [None, '?']:
                input_dbkey = data.dbkey

            identifier = getattr( data, "element_identifier", None )
            if identifier is not None:
                incoming[ "%s|__identifier__" % name ] = identifier

        # Collect chromInfo dataset and add as parameters to incoming
        ( chrom_info, db_dataset ) = app.genome_builds.get_chrom_info( input_dbkey, trans=trans, custom_build_hack_get_len_from_fasta_conversion=tool.id != 'CONVERTER_fasta_to_len' )
        if db_dataset:
            inp_data.update( { "chromInfo": db_dataset } )
        incoming[ "chromInfo" ] = chrom_info

        # Determine output dataset permission/roles list
        existing_datasets = [ inp for inp in inp_data.values() if inp ]
        if existing_datasets:
            output_permissions = app.security_agent.guess_derived_permissions_for_datasets( existing_datasets )
        else:
            # No valid inputs, we will use history defaults
            output_permissions = app.security_agent.history_get_default_permissions( history )

        # Add the dbkey to the incoming parameters
        incoming[ "dbkey" ] = input_dbkey
        # wrapped params are used by change_format action and by output.label; only perform this wrapping once, as needed
        wrapped_params = self._wrapped_params( trans, tool, incoming )

        out_data = odict()
        input_collections = dict( (k, v[0][0]) for k, v in inp_dataset_collections.items() )
        output_collections = OutputCollections(
            trans,
            history,
            tool=tool,
            tool_action=self,
            input_collections=input_collections,
            mapping_over_collection=mapping_over_collection,
            on_text=on_text,
            incoming=incoming,
            params=wrapped_params.params,
            job_params=job_params,
        )

        # Keep track of parent / child relationships, we'll create all the
        # datasets first, then create the associations
        parent_to_child_pairs = []
        child_dataset_names = set()
        object_store_populator = ObjectStorePopulator( app )

        def handle_output( name, output, hidden=None ):
            if output.parent:
                parent_to_child_pairs.append( ( output.parent, name ) )
                child_dataset_names.add( name )
            # What is the following hack for? Need to document under what
            # conditions can the following occur? (james@bx.psu.edu)
            # HACK: the output data has already been created
            #      this happens i.e. as a result of the async controller
            if name in incoming:
                dataid = incoming[name]
                data = trans.sa_session.query( app.model.HistoryDatasetAssociation ).get( dataid )
                assert data is not None
                out_data[name] = data
            else:
                ext = determine_output_format(
                    output,
                    wrapped_params.params,
                    inp_data,
                    inp_dataset_collections,
                    input_ext
                )
                data = app.model.HistoryDatasetAssociation( extension=ext, create_dataset=True, flush=False )
                if hidden is None:
                    hidden = output.hidden
                if hidden:
                    data.visible = False
                trans.sa_session.add( data )
                trans.app.security_agent.set_all_dataset_permissions( data.dataset, output_permissions, new=True )

            # Must flush before setting object store id currently.
            # TODO: optimize this.
            trans.sa_session.flush()
            object_store_populator.set_object_store_id( data )

            # This may not be neccesary with the new parent/child associations
            data.designation = name
            # Copy metadata from one of the inputs if requested.

            # metadata source can be either a string referencing an input
            # or an actual object to copy.
            metadata_source = output.metadata_source
            if metadata_source:
                if isinstance( metadata_source, string_types ):
                    metadata_source = inp_data.get( metadata_source )

            if metadata_source is not None:
                data.init_meta( copy_from=metadata_source )
            else:
                data.init_meta()
            # Take dbkey from LAST input
            data.dbkey = str(input_dbkey)
            # Set state
            data.blurb = "queued"
            # Set output label
            data.name = self.get_output_name( output, data, tool, on_text, trans, incoming, history, wrapped_params.params, job_params )
            # Store output
            out_data[ name ] = data
            if output.actions:
                # Apply pre-job tool-output-dataset actions; e.g. setting metadata, changing format
                output_action_params = dict( out_data )
                output_action_params.update( incoming )
                output.actions.apply_action( data, output_action_params )
            # Also set the default values of actions of type metadata
            self.set_metadata_defaults( output, data, tool, on_text, trans, incoming, history, wrapped_params.params, job_params )
            # Flush all datasets at once.
            return data

        for name, output in tool.outputs.items():
            if not filter_output(output, incoming):
                if output.collection:
                    collections_manager = app.dataset_collections_service
                    element_identifiers = []
                    known_outputs = output.known_outputs( input_collections, collections_manager.type_registry )
                    # Just to echo TODO elsewhere - this should be restructured to allow
                    # nested collections.
                    for output_part_def in known_outputs:
                        # Add elements to top-level collection, unless nested...
                        current_element_identifiers = element_identifiers
                        current_collection_type = output.structure.collection_type

                        for parent_id in (output_part_def.parent_ids or []):
                            # TODO: replace following line with formal abstractions for doing this.
                            current_collection_type = ":".join(current_collection_type.split(":")[1:])
                            name_to_index = dict((value["name"], index) for (index, value) in enumerate(current_element_identifiers))
                            if parent_id not in name_to_index:
                                if parent_id not in current_element_identifiers:
                                    index = len(current_element_identifiers)
                                    current_element_identifiers.append(dict(
                                        name=parent_id,
                                        collection_type=current_collection_type,
                                        src="new_collection",
                                        element_identifiers=[],
                                    ))
                                else:
                                    index = name_to_index[parent_id]
                            current_element_identifiers = current_element_identifiers[ index ][ "element_identifiers" ]

                        effective_output_name = output_part_def.effective_output_name
                        element = handle_output( effective_output_name, output_part_def.output_def, hidden=True )
                        # TODO: this shouldn't exist in the top-level of the history at all
                        # but for now we are still working around that by hiding the contents
                        # there.
                        # Following hack causes dataset to no be added to history...
                        child_dataset_names.add( effective_output_name )

                        history.add_dataset( element, set_hid=set_output_hid, quota=False )
                        trans.sa_session.add( element )
                        trans.sa_session.flush()

                        current_element_identifiers.append({
                            "__object__": element,
                            "name": output_part_def.element_identifier,
                        })
                        log.info(element_identifiers)

                    if output.dynamic_structure:
                        assert not element_identifiers  # known_outputs must have been empty
                        element_kwds = dict(elements=collections_manager.ELEMENTS_UNINITIALIZED)
                    else:
                        element_kwds = dict(element_identifiers=element_identifiers)

                    output_collections.create_collection(
                        output=output,
                        name=name,
                        **element_kwds
                    )
                else:
                    handle_output_timer = ExecutionTimer()
                    handle_output( name, output )
                    log.info("Handled output named %s for tool %s %s" % (name, tool.id, handle_output_timer))

        add_datasets_timer = ExecutionTimer()
        # Add all the top-level (non-child) datasets to the history unless otherwise specified
        datasets_to_persist = []
        for name in out_data.keys():
            if name not in child_dataset_names and name not in incoming:  # don't add children; or already existing datasets, i.e. async created
                data = out_data[ name ]
                datasets_to_persist.append( data )
        # Set HID and add to history.
        # This is brand new and certainly empty so don't worry about quota.
        # TOOL OPTIMIZATION NOTE - from above loop to the job create below 99%+
        # of execution time happens within in history.add_datasets.
        history.add_datasets( trans.sa_session, datasets_to_persist, set_hid=set_output_hid, quota=False, flush=False )

        # Add all the children to their parents
        for parent_name, child_name in parent_to_child_pairs:
            parent_dataset = out_data[ parent_name ]
            child_dataset = out_data[ child_name ]
            parent_dataset.children.append( child_dataset )

        log.info("Added output datasets to history %s" % add_datasets_timer)
        job_setup_timer = ExecutionTimer()
        # Create the job object
        job, galaxy_session = self._new_job_for_session( trans, tool, history )
        self._record_inputs( trans, tool, job, incoming, inp_data, inp_dataset_collections, current_user_roles )
        self._record_outputs( job, out_data, output_collections )
        job.object_store_id = object_store_populator.object_store_id
        if job_params:
            job.params = dumps( job_params )
        job.set_handler(tool.get_job_handler(job_params))
        trans.sa_session.add( job )
        # Now that we have a job id, we can remap any outputs if this is a rerun and the user chose to continue dependent jobs
        # This functionality requires tracking jobs in the database.
        if app.config.track_jobs_in_database and rerun_remap_job_id is not None:
            try:
                old_job = trans.sa_session.query( app.model.Job ).get(rerun_remap_job_id)
                assert old_job is not None, '(%s/%s): Old job id is invalid' % (rerun_remap_job_id, job.id)
                assert old_job.tool_id == job.tool_id, '(%s/%s): Old tool id (%s) does not match rerun tool id (%s)' % (old_job.id, job.id, old_job.tool_id, job.tool_id)
                if trans.user is not None:
                    assert old_job.user_id == trans.user.id, '(%s/%s): Old user id (%s) does not match rerun user id (%s)' % (old_job.id, job.id, old_job.user_id, trans.user.id)
                elif trans.user is None and type( galaxy_session ) == trans.model.GalaxySession:
                    assert old_job.session_id == galaxy_session.id, '(%s/%s): Old session id (%s) does not match rerun session id (%s)' % (old_job.id, job.id, old_job.session_id, galaxy_session.id)
                else:
                    raise Exception('(%s/%s): Remapping via the API is not (yet) supported' % (old_job.id, job.id))
                # Duplicate PJAs before remap.
                for pjaa in old_job.post_job_actions:
                    job.add_post_job_action(pjaa.post_job_action)
                for jtod in old_job.output_datasets:
                    for (job_to_remap, jtid) in [(jtid.job, jtid) for jtid in jtod.dataset.dependent_jobs]:
                        if (trans.user is not None and job_to_remap.user_id == trans.user.id) or (trans.user is None and job_to_remap.session_id == galaxy_session.id):
                            if job_to_remap.state == job_to_remap.states.PAUSED:
                                job_to_remap.state = job_to_remap.states.NEW
                            for hda in [ dep_jtod.dataset for dep_jtod in job_to_remap.output_datasets ]:
                                if hda.state == hda.states.PAUSED:
                                    hda.state = hda.states.NEW
                                    hda.info = None
                            input_values = dict( [ ( p.name, json.loads( p.value ) ) for p in job_to_remap.parameters ] )
                            update_param( jtid.name, input_values, str( out_data[ jtod.name ].id ) )
                            for p in job_to_remap.parameters:
                                p.value = json.dumps( input_values[ p.name ] )
                            jtid.dataset = out_data[jtod.name]
                            jtid.dataset.hid = jtod.dataset.hid
                            log.info('Job %s input HDA %s remapped to new HDA %s' % (job_to_remap.id, jtod.dataset.id, jtid.dataset.id))
                            trans.sa_session.add(job_to_remap)
                            trans.sa_session.add(jtid)
                    jtod.dataset.visible = False
                    trans.sa_session.add(jtod)
            except Exception:
                log.exception('Cannot remap rerun dependencies.')

        log.info("Setup for job %s complete, ready to flush %s" % (job.log_str(), job_setup_timer))

        job_flush_timer = ExecutionTimer()
        trans.sa_session.flush()
        log.info("Flushed transaction for job %s %s" % (job.log_str(), job_flush_timer))
        # Some tools are not really executable, but jobs are still created for them ( for record keeping ).
        # Examples include tools that redirect to other applications ( epigraph ).  These special tools must
        # include something that can be retrieved from the params ( e.g., REDIRECT_URL ) to keep the job
        # from being queued.
        if 'REDIRECT_URL' in incoming:
            # Get the dataset - there should only be 1
            for name in inp_data.keys():
                dataset = inp_data[ name ]
            redirect_url = tool.parse_redirect_url( dataset, incoming )
            # GALAXY_URL should be include in the tool params to enable the external application
            # to send back to the current Galaxy instance
            GALAXY_URL = incoming.get( 'GALAXY_URL', None )
            assert GALAXY_URL is not None, "GALAXY_URL parameter missing in tool config."
            redirect_url += "&GALAXY_URL=%s" % GALAXY_URL
            # Job should not be queued, so set state to ok
            job.set_state( app.model.Job.states.OK )
            job.info = "Redirected to: %s" % redirect_url
            trans.sa_session.add( job )
            trans.sa_session.flush()
            trans.response.send_redirect( url_for( controller='tool_runner', action='redirect', redirect_url=redirect_url ) )
        else:
            # Put the job in the queue if tracking in memory
            app.job_queue.put( job.id, job.tool_id )
            trans.log_event( "Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id )
            return job, out_data

    def _wrapped_params( self, trans, tool, incoming ):
        wrapped_params = WrappedParameters( trans, tool, incoming )
        return wrapped_params

    def _get_on_text( self, inp_data ):
        input_names = []
        for name, data in reversed( inp_data.items() ):
            if getattr( data, "hid", None ):
                input_names.append( 'data %s' % data.hid )

        return on_text_for_names( input_names )

    def _new_job_for_session( self, trans, tool, history ):
        job = trans.app.model.Job()
        galaxy_session = None

        if hasattr( trans, "get_galaxy_session" ):
            galaxy_session = trans.get_galaxy_session()
            # If we're submitting from the API, there won't be a session.
            if type( galaxy_session ) == trans.model.GalaxySession:
                job.session_id = galaxy_session.id
        if trans.user is not None:
            job.user_id = trans.user.id
        job.history_id = history.id
        job.tool_id = tool.id
        try:
            # For backward compatibility, some tools may not have versions yet.
            job.tool_version = tool.version
        except:
            job.tool_version = "1.0.0"
        return job, galaxy_session

    def _record_inputs( self, trans, tool, job, incoming, inp_data, inp_dataset_collections, current_user_roles ):
        # FIXME: Don't need all of incoming here, just the defined parameters
        #        from the tool. We need to deal with tools that pass all post
        #        parameters to the command as a special case.
        for name, dataset_collection_info_pairs in inp_dataset_collections.items():
            first_reduction = True
            for ( dataset_collection, reduced ) in dataset_collection_info_pairs:
                # TODO: update incoming for list...
                if reduced and first_reduction:
                    first_reduction = False
                    incoming[ name ] = []
                if reduced:
                    incoming[ name ].append( { 'id': dataset_collection.id, 'src': 'hdca' } )
                # Should verify security? We check security of individual
                # datasets below?
                # TODO: verify can have multiple with same name, don't want to loose tracability
                job.add_input_dataset_collection( name, dataset_collection )
        for name, value in tool.params_to_strings( incoming, trans.app ).items():
            job.add_parameter( name, value )
        self._check_input_data_access( trans, job, inp_data, current_user_roles )

    def _record_outputs( self, job, out_data, output_collections ):
        out_collections = output_collections.out_collections
        out_collection_instances = output_collections.out_collection_instances
        for name, dataset in out_data.items():
            job.add_output_dataset( name, dataset )
        for name, dataset_collection in out_collections.items():
            job.add_implicit_output_dataset_collection( name, dataset_collection )
        for name, dataset_collection_instance in out_collection_instances.items():
            job.add_output_dataset_collection( name, dataset_collection_instance )

    def _check_input_data_access( self, trans, job, inp_data, current_user_roles ):
        access_timer = ExecutionTimer()
        for name, dataset in inp_data.items():
            if dataset:
                if not trans.app.security_agent.can_access_dataset( current_user_roles, dataset.dataset ):
                    raise Exception("User does not have permission to use a dataset (%s) provided for input." % dataset.id)
                if dataset in trans.sa_session:
                    job.add_input_dataset( name, dataset=dataset )
                else:
                    job.add_input_dataset( name, dataset_id=dataset.id )
            else:
                job.add_input_dataset( name, None )
        job_str = job.log_str()
        log.info("Verified access to datasets for %s %s" % (job_str, access_timer))

    def get_output_name( self, output, dataset, tool, on_text, trans, incoming, history, params, job_params ):
        if output.label:
            params['tool'] = tool
            params['on_string'] = on_text
            return fill_template( output.label, context=params )
        else:
            return self._get_default_data_name( dataset, tool, on_text=on_text, trans=trans, incoming=incoming, history=history, params=params, job_params=job_params )

    def set_metadata_defaults( self, output, dataset, tool, on_text, trans, incoming, history, params, job_params ):
        """
        This allows to map names of input files to metadata default values. Example:

        <data format="tabular" name="output" label="Tabular output, aggregates data from individual_inputs" >
            <actions>
                <action name="column_names" type="metadata" default="${','.join([input.name for input in $individual_inputs ])}" />
            </actions>
        </data>
        """
        if output.actions:
            for action in output.actions.actions:
                if action.tag == "metadata" and action.default:
                    metadata_new_value = fill_template( action.default, context=params ).split(",")
                    dataset.metadata.__setattr__(str(action.name), metadata_new_value)

    def _get_default_data_name( self, dataset, tool, on_text=None, trans=None, incoming=None, history=None, params=None, job_params=None, **kwd ):
        name = tool.name
        if on_text:
            name += ( " on " + on_text )
        return name

class ObjectStorePopulator( object ):
    """ Small helper for interacting with the object store and making sure all
    datasets from a job end up with the same object_store_id.
    """

    def __init__( self, app ):
        self.object_store = app.object_store
        self.object_store_id = None

    def set_object_store_id( self, data ):
        # Create an empty file immediately.  The first dataset will be
        # created in the "default" store, all others will be created in
        # the same store as the first.
        data.dataset.object_store_id = self.object_store_id
        try:
            self.object_store.create( data.dataset )
        except ObjectInvalid:
            raise Exception('Unable to create output dataset: object store is full')
        self.object_store_id = data.dataset.object_store_id  # these will be the same thing after the first output

class OutputCollections(object):
    """ Keeps track of collections (DC or HDCA) created by actions.

    Actions do fairly different things depending on whether we are creating
    just part of an collection or a whole output collection (mapping_over_collection
    parameter).
    """

    def __init__(self, trans, history, tool, tool_action, input_collections, mapping_over_collection, on_text, incoming, params, job_params):
        self.trans = trans
        self.history = history
        self.tool = tool
        self.tool_action = tool_action
        self.input_collections = input_collections
        self.mapping_over_collection = mapping_over_collection
        self.on_text = on_text
        self.incoming = incoming
        self.params = params
        self.job_params = job_params
        self.out_collections = {}
        self.out_collection_instances = {}

    def create_collection(self, output, name, **element_kwds):
        input_collections = self.input_collections
        collections_manager = self.trans.app.dataset_collections_service
        collection_type = output.structure.collection_type
        if collection_type is None:
            collection_type_source = output.structure.collection_type_source
            if collection_type_source is None:
                # TODO: Not a new problem, but this should be determined
                # sooner.
                raise Exception("Could not determine collection type to create.")
            if collection_type_source not in input_collections:
                raise Exception("Could not find collection type source with name [%s]." % collection_type_source)

            collection_type = input_collections[collection_type_source].collection.collection_type

        if self.mapping_over_collection:
            dc = collections_manager.create_dataset_collection(
                self.trans,
                collection_type=collection_type,
                **element_kwds
            )
            self.out_collections[ name ] = dc
        else:
            hdca_name = self.tool_action.get_output_name(
                output,
                None,
                self.tool,
                self.on_text,
                self.trans,
                self.incoming,
                self.history,
                self.params,
                self.job_params,
            )
            hdca = collections_manager.create(
                self.trans,
                self.history,
                name=hdca_name,
                collection_type=collection_type,
                trusted_identifiers=True,
                **element_kwds
            )
            # name here is name of the output element - not name
            # of the hdca.
            self.out_collection_instances[ name ] = hdca

def on_text_for_names( input_names ):
    # input_names may contain duplicates... this is because the first value in
    # multiple input dataset parameters will appear twice once as param_name
    # and once as param_name1.
    unique_names = []
    for name in input_names:
        if name not in unique_names:
            unique_names.append( name )
    input_names = unique_names

    # Build name for output datasets based on tool name and input names
    if len( input_names ) == 1:
        on_text = input_names[0]
    elif len( input_names ) == 2:
        on_text = '%s and %s' % tuple(input_names[0:2])
    elif len( input_names ) == 3:
        on_text = '%s, %s, and %s' % tuple(input_names[0:3])
    elif len( input_names ) > 3:
        on_text = '%s, %s, and others' % tuple(input_names[0:2])
    else:
        on_text = ""
    return on_text

def filter_output(output, incoming):
    for filter in output.filters:
        try:
            if not eval( filter.text.strip(), globals(), incoming ):
                return True  # do not create this dataset
        except Exception as e:
            log.debug( 'Dataset output filter failed: %s' % e )
    return False

def determine_output_format(output, parameter_context, input_datasets, input_dataset_collections, random_input_ext):
    """ Determines the output format for a dataset based on an abstract
    description of the output (galaxy.tools.parser.ToolOutput), the parameter
    wrappers, a map of the input datasets (name => HDA), and the last input
    extensions in the tool form.

    TODO: Don't deal with XML here - move this logic into ToolOutput.
    TODO: Make the input extension used deterministic instead of random.
    """
    # the type should match the input
    ext = output.format
    if ext == "input":
        ext = random_input_ext
    format_source = output.format_source
    if format_source is not None and format_source in input_datasets:
        try:
            input_dataset = input_datasets[output.format_source]
            input_extension = input_dataset.ext
            ext = input_extension
        except Exception:
            pass
    elif format_source is not None:
        if re.match(r"^[^\[\]]*\[[^\[\]]*\]$", format_source):
            collection_name, element_index = format_source[0:-1].split("[")
            # Treat as json to interpret "forward" vs 0 with type
            # Make it feel more like Python, single quote better in XML also.
            element_index = element_index.replace("'", '"')
            element_index = json.loads(element_index)

            if collection_name in input_dataset_collections:
                try:
                    input_collection = input_dataset_collections[collection_name][0][0]
                    input_dataset = input_collection.collection[element_index].element_object
                    input_extension = input_dataset.ext
                    ext = input_extension
                except Exception as e:
                    log.debug("Exception while trying to determine format_source: %s", e)
                    pass

    # process change_format tags
    if output.change_format is not None:
        new_format_set = False
        for change_elem in output.change_format:
            for when_elem in change_elem.findall( 'when' ):
                check = when_elem.get( 'input', None )
                if check is not None:
                    try:
                        if '$' not in check:
                            # allow a simple name or more complex specifications
                            check = '${%s}' % check
                        if str( fill_template( check, context=parameter_context ) ) == when_elem.get( 'value', None ):
                            ext = when_elem.get( 'format', ext )
                    except:  # bad tag input value; possibly referencing a param within a different conditional when block or other nonexistent grouping construct
                        continue
                else:
                    check = when_elem.get( 'input_dataset', None )
                    if check is not None:
                        check = input_datasets.get( check, None )
                        # At this point check is a HistoryDatasetAssociation object.
                        check_format = when_elem.get( 'format', ext )
                        check_value = when_elem.get( 'value', None )
                        check_attribute = when_elem.get( 'attribute', None )
                        if check is not None and check_value is not None and check_attribute is not None:
                            # See if the attribute to be checked belongs to the HistoryDatasetAssociation object.
                            if hasattr( check, check_attribute ):
                                if str( getattr( check, check_attribute ) ) == str( check_value ):
                                    ext = check_format
                                    new_format_set = True
                                    break
                            # See if the attribute to be checked belongs to the metadata associated with the
                            # HistoryDatasetAssociation object.
                            if check.metadata is not None:
                                metadata_value = check.metadata.get( check_attribute, None )
                                if metadata_value is not None:
                                    if str( metadata_value ) == str( check_value ):
                                        ext = check_format
                                        new_format_set = True
                                        break
            if new_format_set:
                break
    return ext


#!/usr/bin/env python
'''
    Code to read a galaxy, random catalog,
    swap weights, subsample randoms, assess the redshift distribution
    export the new catalogs

    get help with $> python swap_data.py --help
    run with $> python swap_data.py
'''
import os
import logging

import sys
sys.path.append('/home/mehdi/github/LSSutils')
from LSSutils.catalogs.combinefits import SysWeight, EbossCatalog, reassignment

def main(model='plain',
         zmin=0.8,
         zmax=3.5,
         nside=512,
         zsplit='lowmidhigh',
         slices=['low', 'high', 'zhigh'],
         cap='NGC',
         target='QSO',
         version='v7_2',
         versiono='0.1'):

    output_dir    = f'/B/Shared/mehdi/eboss/data/{version}/{versiono}/'
    data_name_in  = f'/B/Shared/mehdi/eboss/data/{version}/eBOSS_{target}_full_{cap}_{version}.dat.fits'
    rand_name_in  = f'/B/Shared/mehdi/eboss/data/{version}/eBOSS_{target}_full_{cap}_{version}.ran.fits'

    tag           = '_'.join((version, versiono, model, zsplit))
    data_name_out = output_dir + f'eBOSS_{target}_clustering_{cap}_{tag}.dat.fits'
    rand_name_out = output_dir + f'eBOSS_{target}_clustering_{cap}_{tag}.ran.fits'
    plotname      = output_dir + f'eBOSS_{target}_{cap}_{tag}.pdf'

    weight = lambda zcut, model: output_dir + f'/results/{cap}_{zcut}_{nside}'\
             +f'/regression/nn_{model}/nn-weights.hp{nside}.fits'

    zcuts = {'low':[[0.8, 1.5],   None],
             'high':[[1.5, 2.2],  None],
             'all':[[0.8, 2.2],   None],
             'zhigh':[[2.2, 3.5], None],
             'z1':[[0.8, 1.3], None],
             'z2':[[1.3, 1.6], None],
             'z3':[[1.6, 2.2], None],
             'all_racut':[[0.8, 2.2], None],
             'zhigh_racut':[[2.2, 3.5], None]}

    logger = logging.getLogger("Swapper")
    logger.info('results will be written under {}'.format(output_dir))
    logger.info('swap the NN-z weights')
    logger.info(f'input data   : {data_name_in}')
    logger.info(f'input random : {rand_name_in}')
    logger.info(f'output data   : {data_name_out}')
    logger.info(f'output random : {rand_name_out}')

    # --- check if the output directory exists
    if not os.path.isdir(output_dir):
        logger.info('create {}'.format(output_dir))
        #os.makedirs(output_dir)

    for zcut in slices:
        logger.info(f'zcut : {zcut}')
        zcuts[zcut][1]=SysWeight(weight(zcut, model))

    data = EbossCatalog(data_name_in, zmin=zmin, zmax=zmax, kind='galaxy')
    data.swap(zcuts=zcuts, slices=slices)
    #data.make_plots(zcuts, slices=slices, filename=plotname)
    data.to_fits(data_name_out)

    random    = EbossCatalog(rand_name_in, zmin=zmin, zmax=zmax, kind='random')
    newrandom = reassignment(random.data, data.data)
    newrandom.write(rand_name_out)

if __name__ == '__main__':

    from argparse import ArgumentParser
    ap = ArgumentParser(description='Prepare EBOSS Data and Random Catalogs')
    ap.add_argument('-m','--model',   type=str,   default='plain', help='eg:plain, other options are ablation and known ')
    ap.add_argument('--zmin',    type=float, default=0.8, help='eg:0.8')
    ap.add_argument('--zmax',    type=float, default=3.5, help='eg:3.5')
    ap.add_argument('-n', '--nside',   type=int,   default=512, help='eg:512')
    ap.add_argument('-zs', '--zsplit',  type=str,   default='lowmidhigh', help='eg: lowmidhigh')
    ap.add_argument('-sl', '--slices',  type=str,   default=['low', 'high', 'zhigh'], nargs='*', help="eg:['low', 'high', 'zhigh']")
    ap.add_argument('-c', '--cap',     type=str,   default='NGC', help='eg: NGC or SGC')
    ap.add_argument('-t', '--target',  type=str,   default='QSO', help='eg: QSO')
    ap.add_argument('-v', '--version', type=str,   default='v7_2', help='eg: v7_2')
    ap.add_argument('-vo', '--versiono',type=str,   default='0.3', help='eg: 0.3')
    ns = ap.parse_args()

    #--- default
    #
    # model='plain',
    # zmin=0.8,
    # zmax=3.5,
    # nside=512,
    # zsplit='lowmidhigh',
    # slices=['low', 'high', 'zhigh'],
    # cap='NGC',
    # target='QSO',
    # version='v7_2',
    # versiono='0.3'

    from LSSutils import setup_logging
    setup_logging('info')

    logger = logging.getLogger("Swapper")

    kwargs = ns.__dict__
    for (a,b) in zip(kwargs.keys(), kwargs.values()):
        logger.info('{:6s}{:15s} : {}'.format('', a, b))

    # -- call the function
    main(**kwargs)


import argparse
import winreg
from datetime import datetime, timedelta
import struct
import io

"""
This code is designed to query an active Windows Registry for key values and artifacts that
investigators may find useful. This code can be used to gather a rudimentary baseline of a
registry or to hunt for abnormal keys and values within certain registries on a LIVE system.

Usage: LiveRegistryEnumeration.py [--hive $VAR1] [--wireless] [--usb] [--shimcache]

"""
#Setup the command line arguments and help messages
parser = argparse.ArgumentParser(description="Live Registry Analysis:\nThis program provides some options for quick registry analysis to include Run Key Analysis, Wireless Network listings, USB information,and SHIM CACHE information.")
parser.add_argument('--hive', type=str, help='Display Run Keys. Options are: \'HKLM\', \'HKCU\', or \'Both\'')
parser.add_argument('-w', '--wireless', help='List Wireless Networks(Admin)', action="store_true")
parser.add_argument('-u', '--usb', help='List USB Devices', action="store_true")
parser.add_argument('--shimcache', type=int, help='(Experimental) Show logged data starting at given number of days ago.')
args = parser.parse_args()

def checkRunKeys(hive, name):
	#Check the Run and RunOnce keys in the given Hive and print results to screen
	#Results will be in the form 'Program Name: Program Path of Execution'
	keys = ['SOFTWARE\Microsoft\Windows\CurrentVersion\Run', 'SOFTWARE\Microsoft\Windows\CurrentVersion\RunOnce']
	for k in keys:
		checkKey = winreg.CreateKey(hive, k)
		print ("Attempting enumeration of {0}\\{1}".format(name, k))
		print ("--- Program Name: Command Line")
		numValues = winreg.QueryInfoKey(checkKey)[1]
		if numValues > 0:
		  for index in range(numValues):
			  nameKey = winreg.EnumValue(checkKey, index)
			  print ("--- {0}: {1}".format(nameKey[0], nameKey[1]))
		  print ("Key Complete\n")
		else:
			print ("No Values in this Key\n")

def wirelessNetworks():
	#Check for all recorded connections to wireless networks and print results to the screen
	#Results will be in the form 'DNS Name: SSID'
	print ("Attempting enumeration of HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\NetworkList\\Signatures\\Unmanaged")
	print ("--- DNS Suffix: SSID")
	checkKey = winreg.CreateKey(winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\NetworkList\\Signatures\\Unmanaged')
	numValues = winreg.QueryInfoKey(checkKey)[0]
	if numValues > 0:
		for index in range(numValues):
			nameKey = winreg.EnumKey(checkKey, index)
			trueKey = winreg.CreateKey(winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\NetworkList\\Signatures\\Unmanaged\\' + nameKey)
			DnsSuffix = winreg.EnumValue(trueKey, 3)
			FirstNetwork = winreg.EnumValue(trueKey, 4)
			print ("--- {0}: {1}".format(DnsSuffix[1], FirstNetwork[1]))
		print ("No additional Wireless Network Connections Found\n")
	else:
		print ("No Values in this Key\n")

def usbDevices():
	#Check for all recorded USB Devices that have connected to the system and print results to the screen
	#Results will be in the form 'Last Key Modification Date(UTC) - Friendly Name: ContainerID'
	print ("Attempting enumeration of USB Storage Devices")
	print ("--- Date First Connected(UTC) - Friendly Name: ContainerID")
	checkKey = winreg.CreateKey(winreg.HKEY_LOCAL_MACHINE, 'SYSTEM\\CurrentControlSet\\Enum\\USBSTOR')
	numValues = winreg.QueryInfoKey(checkKey)[0]
	if numValues > 0:
		for index in range(numValues):
			nextKey = winreg.CreateKey(winreg.HKEY_LOCAL_MACHINE, 'SYSTEM\\CurrentControlSet\\Enum\\USBSTOR\\' + winreg.EnumKey(checkKey, index))
			trueKey = winreg.CreateKey(winreg.HKEY_LOCAL_MACHINE, 'SYSTEM\\CurrentControlSet\\Enum\\USBSTOR\\' + winreg.EnumKey(checkKey, index) + '\\' + winreg.EnumKey(nextKey, 0))
			#For some reason, some keys index values differently. We must iterate through all values to guarantee accuracy as a result.
			for value in range(winreg.QueryInfoKey(trueKey)[1]):
				curValue = winreg.EnumValue(trueKey, value)[0]
				nanoseconds = winreg.QueryInfoKey(trueKey)[2]
				if curValue == 'ContainerID':
					ContainerID = winreg.EnumValue(trueKey, value)
				elif curValue == 'FriendlyName':
					FriendlyName = winreg.EnumValue(trueKey, value)
			print ("--- {2} - {0}: {1}".format(FriendlyName[1], ContainerID[1], translateTime(nanoseconds)))
		print ("No additional devices found in USBSTOR\n")
	else:
		print ("No values in this key.")

def readShimCache(daysOfData):
	#Check for all programs recorded in the App Compatibility Cache (shim cache).
	#Results are in the form: Modification date : Path
	#What about execution flag?
	print ("(Experimental) Attempting enumeration of the SHIM Cache (Experimental)")
	print ("--- UTC Modification Time : Path")
	try:
	  checkKey = winreg.CreateKey(winreg.HKEY_LOCAL_MACHINE, 'SYSTEM\\CurrentControlSet\\Control\\Session Manager\\AppCompatCache')
	except:
	  print ("Shim Cache not in expected Key. Other Keys and OS Version Support under development.\n")
	entrylist = []
	shim = winreg.EnumValue(checkKey, 0)[1]
	cache_data = shim[0x34:]
	data = io.BytesIO(cache_data)
	while data.tell() < len(cache_data):
		header = data.read(12)
		magic, crc32_hash, entry_len = struct.unpack('<4sLL', header)
		entry_data = io.BytesIO(data.read(entry_len))
		path_len = struct.unpack('<H', entry_data.read(2))[0]
		if path_len == 0:
			path = 'None'
		else:
			path = entry_data.read(path_len).decode('utf-16le', 'replace').encode('utf-8')
		low_datetime, high_datetime = struct.unpack('<LL', entry_data.read(8))
		adjustedTimes = (high_datetime << 32) | low_datetime
		#Windows uses January 1, 1601 as start time. Get number of microseconds from then to given number of days ago.
		timeRange = (timedelta(days=(152539-daysOfData)).total_seconds()) * 1000000
		if adjustedTimes / 10 > timeRange:
			row = [adjustedTimes, path.decode()]
			entrylist.append(row)
		else:
			continue
  #Need to sort dates appropriately as time-order is not maintained in the cache
	return sorted(entrylist)

def translateTime(nanoseconds):
	#Windows timestamps are number of 100ns since January 1st, 1601. This function translates time to make it more readable.
	return format(datetime(1601,1,1) + timedelta(microseconds=nanoseconds/10), '%d %B %Y %H:%M:%S')

def main():
	if args.hive is None and not args.wireless and not args.usb and args.shimcache is None:
		print ("No options given! Add -h to the command to display options!\n")
	else:
		if args.hive is not None:
			if args.hive.upper() in ("HKLM", "HKCU", "BOTH"):
				print ("--------------RUN KEYS--------------")
				if args.hive.upper() == "HKLM":
					checkRunKeys(winreg.HKEY_LOCAL_MACHINE, "HKEY_LOCAL_MACHINE")
				elif args.hive.upper() == "HKCU":
					checkRunKeys(winreg.HKEY_CURRENT_USER, "HKEY_CURRENT_USER")
				elif args.hive.lower() == "both":
					checkRunKeys(winreg.HKEY_LOCAL_MACHINE, "HKEY_LOCAL_MACHINE")
					checkRunKeys(winreg.HKEY_CURRENT_USER, "HKEY_CURRENT_USER")
			else:
				print ("Unrecongnized Choice for HIVE: \"{}\" is not valid.\n".format(args.hive))
		if args.wireless:
			print ("--------------WIRELESS NETWORK CONNECTIONS--------------")
			try:
				wirelessNetworks()
			except PermissionError:
				print ("You must be admin to check this key.\n")
		if args.usb:
			print ("--------------USB DEVICES--------------")
			usbDevices()
		if args.shimcache is not None:
			print ("--------------SHIM CACHE DATA--------------")
			results = readShimCache(args.shimcache)
			for entry in results:
				if entry[0] == 0:
					entry[0] = "No Timestamp - Unknown"
				else:
					entry[0] = translateTime(entry[0])
				print ("--- {0} : {1}".format(entry[0], entry[1]))
			print ("End of SHIM Cache Information\n")
	print ("Execution Complete")

if __name__ == "__main__":
  main()


import math
import pygame.mixer
from twisted.internet import defer
from twisted.spread import pb
from vector import Vector2D
from twisted.internet import reactor

#This comment is still not important.
sounds = dict()

def _initSounds():
#    pygame.mixer.init(frequency=16000)#, size=-8, channels=1)
    sounds["trigger trap"] = pygame.mixer.Sound("data/sfx/alex_sfx/Trigger Trap.wav")
    sounds["explosion"] = pygame.mixer.Sound("data/sfx/alex_sfx/Attack Hit.wav")

    sounds["attack"] = pygame.mixer.Sound("data/sfx/alex_sfx/attack.wav")
    sounds["poly armor full"] = pygame.mixer.Sound("data/sfx/alex_sfx/Points Full.wav")
    sounds["player upgrade"] = pygame.mixer.Sound("data/sfx/alex_sfx/You upgraded.wav")

    sounds["accept upgrade"] = pygame.mixer.Sound("data/sfx/alex_sfx/accept_upgrade.wav")

    sounds["gain poly armor"] = pygame.mixer.Sound("data/sfx/alex_sfx/gain resource.wav")
    sounds["lose poly armor"] = pygame.mixer.Sound("data/sfx/alex_sfx/pay resource.wav")
    sounds["poly armor depleted"] = pygame.mixer.Sound("data/sfx/alex_sfx/resources depleted.wav")

    sounds["mining"] = pygame.mixer.Sound("data/sfx/alex_sfx/In Resource Pool(loop).wav")

    sounds["building",3] = pygame.mixer.Sound("data/sfx/alex_sfx/Building 3-sided.wav")
    sounds["building",4] = pygame.mixer.Sound("data/sfx/alex_sfx/Building 4-sided.wav")
    sounds["building",5] = pygame.mixer.Sound("data/sfx/alex_sfx/Building 5-sided.wav")

    sounds["finish building",3] = pygame.mixer.Sound("data/sfx/alex_sfx/Finish 3-sided.wav")
    sounds["finish building",4] = pygame.mixer.Sound("data/sfx/alex_sfx/Finish 4-sided.wav")
    sounds["finish building",5] = pygame.mixer.Sound("data/sfx/alex_sfx/Finish 5-sided.wav")

    sounds["scanning"] = pygame.mixer.Sound("data/sfx/alex_sfx/Sweeping.wav")

def getSound(strIdx, nIndex = None):
    if not nIndex == None:
        return sounds[strIdx, nIndex]
    else:
        return sounds[strIdx]

class PlayerScan:
    def __init__(self):
        self.reset()

    def reset(self):
        self.startTime = 0
        self._radius = 0
        self.resetTimer = None
        self._isScanning = False

    def start(self):
        if self.resetTimer:
            self.resetTimer.cancel()
            self.reset()
        self.startTime = pygame.time.get_ticks()
        self._isScanning = True

    def stop(self):
        self._isScanning = False
        self._radius = self.radius()
        self.startTime = pygame.time.get_ticks()
        self.resetTimer = reactor.callLater(5, self.reset)

    def radius(self):
        if self.startTime == 0:
            return 0
        dt = (pygame.time.get_ticks() - self.startTime)
        if self._radius:
            if dt >= 5000.0:
                return 0;
            else:
                return self._radius * (1 - (dt / 5000.0))
        #return (math.log1p(min(1, (dt / 10000.0) / (math.e - 1))) * .9) + 0.1
        return  2*(min(1, (dt / 1000.0)  * .9) + 0.1)

    def __nonzero__(self):
        if self.startTime == 0:
            return False
        return True

    def isScanning(self):
        return self._isScanning

class Player(pb.Cacheable, pb.RemoteCache):
    def __init__(self):
        #pb.Cacheable.__init__(self)
        #pb.RemoteCache.__init__(self)
        self.position = Vector2D(0, 0)
        self.sides = 3
        self.resources = 0
        self.observers = []
        self.scanning = PlayerScan()
        self.size = 1
        self.action = None
        self.upgradingAt = None
        self.self = False
        self.events = set()
        self.topEvents = set()
        self.armor = dict()
        self.building = None
        self._buildingReset = None
        self.tooltip = None
        self.lastAction = None
        #self.sounds = None

        #sound related state
        self.playingBuildingCompleteSound = False
        self.scanFadeOutOk = False
        self.stopBuildingChannelOk = True
        self.hadResources = False
        #self.pointsFullPlayOk = True
        #self.sounds = dict()
        #self.sounds['Building 3-Sided'] = pygame.mixer.Sound("data/sfx/alex_sfx/Building 3-sided.wav")

        #sound related state
        self.playingBuildingCompleteSound = False
        self.actionName = None
        self.scanFadeOutOk = False
        self.stopBuildingChannelOk = True
        self.attacking = False
        self.miningAnimation = None

    def reset(self):
        self.setResources(0)
        self.sides = 3

#        self.resources = 0
#        self.size = 1
#        self.action = None
#        self.upgradingAt = None
#        self.events = set()
#        self.topEvents = set()
#        self.armor = dict()
#        self.building = None
#        self._buildingReset = None
#        self.tooltip = None
#        self.lastAction = None
#        #self.sounds = None
#
#        #sound related state
#        self.playingBuildingCompleteSound = False
#        self.scanFadeOutOk = False
#        self.stopBuildingChannelOk = True
#        self.hadResources = False
#        #self.pointsFullPlayOk = True
#        #self.sounds = dict()
#        #self.sounds['Building 3-Sided'] = pygame.mixer.Sound("data/sfx/alex_sfx/Building 3-sided.wav")
#
#        #sound related state
#        self.playingBuildingCompleteSound = False
#        self.actionName = None
#        self.scanFadeOutOk = False
#        self.stopBuildingChannelOk = True
#        self.attacking = False
#        self.miningAnimation = None

    def _startScanning(self):
        self.scanning.start()

    def startScanning(self):
        self._startScanning()
        for o in self.observers: o.callRemote('startScanning')

    observe_startScanning = _startScanning

    def _finishScanning(self):
        self.scanning.stop()
    def finishScanning(self):
        self._finishScanning()
        for o in self.observers: o.callRemote('finishScanning')
    observe_finishScanning = _finishScanning

    def getScanRadius(self):
        return self.scanning.radius()

    def observe_trapped(self, playSound = True):
        if self.resources:
            self.setResources(0)
        else:
            self.sides = 0

        if (playSound):
            pygame.mixer.Channel(6).play(getSound("trigger trap"))
            pygame.mixer.Channel(7).play(getSound("explosion"))

    def trapped(self):
        self.observe_trapped(playSound = False)
        for o in self.observers: o.callRemote('trapped')

    def setAction(self, remote, local):
        self.observe_setAction(remote)
        self.action = local
        self.actionName = remote
        if (remote != "Mining" and remote != "Building"):
            pygame.mixer.Channel(7).stop()

#        if(remote == "Mining"):
#            self.miningAnimation = self.images["mining"].copy()
#            self.miningAnimation.start()
            #self.events.add(self.miningAnimation)
            #self.miningAnimation.draw(view.screen, position)

        for o in self.observers: o.callRemote('setAction', remote)

    def observe_setAction(self, action):
        # TODO Tooltips no longer used?
        self.tooltip = None
        self.actionName = action

    def _gainResource(self, playSound = True):
        playResourceFullOk = False
        actuallyGainResource = False

        if self.sides < 3:
            self.sides += 1
            if playSound:
                pygame.mixer.Channel(6).play(getSound("player upgrade"))
                animation = self.images["player upgraded"].copy()
                animation.start(12).addCallback(lambda ign: self.events.remove(animation))
                self.events.add(animation)
            #TODO should probably play some kind of sound here
        elif self.resources < self.sides:

            #self.resources += 1
            self.setResources(self.resources + 1)

            actuallyGainResource = True

            #animation = self.images["Generic_2"].copy()
            #animation.start(12).addCallback(lambda ign: self.events.remove(animation))
            #self.events.add(animation)

            playResourceFullOk = True

        if (playSound):
            if actuallyGainResource:
                #pygame.mixer.Channel(6).play(pygame.mixer.Sound("data/sfx/alex_sfx/gain resource.wav"))
                pygame.mixer.Channel(6).play(getSound("gain poly armor"))

                #It's possible if the sound changes, that restarting the mining sound will sound good
                #pygame.mixer.Channel(7).play(pygame.mixer.Sound("data/sfx/alex_sfx/In Resource Pool(loop).wav"))
                pygame.mixer.Channel(7).play(getSound("mining"))

            if self.resources == self.sides:
                pygame.mixer.Channel(7).stop()
                if (playResourceFullOk):
                    self.stopBuildingChannelOk = False
                    pygame.mixer.Channel(5).play(getSound("poly armor full"))

    def gainResource(self):
        self._gainResource(playSound = False)

        for o in self.observers: o.callRemote('gainResource')

    observe_gainResource = _gainResource

    def switchTeams(self):
        self.observe_switchTeams()
        for o in self.observers: o.callRemote('switchTeams')

    def observe_switchTeams(self):
        #print "team: " + str(self.team)
        if self.team == 1:
            self.team = 2
        else:
            self.team = 1

    def setResources(self, newAmount):
        self.resources = newAmount
        self.armor.clear()

        for i in range(1, newAmount + 1):
            self.armor[i] = self.images["Armor", self.sides, i]

    def _loseResource(self, playSound = True):
        if self.resources:
            if self.building:
                infiniteResources = False
                if not infiniteResources:
                    self.setResources(self.resources - 1)

            if playSound:
                #TODO building complete sounds should be played here
                if not self.building:
                    pygame.mixer.Channel(7).stop()
                else:
                    pygame.mixer.Channel(6).play(getSound("lose poly armor"))
                    #print "the building : " + str(self.building.sides) + "-sided, resources =" + str(self.building.resources) + "\n"
                    if self.building.resources == 0:#self.building.nResourcesToUpgrade() == 1:#self.building.sides == self.building.resources and self.building.sides >= 3:
                        pygame.mixer.Channel(7).stop()
                        self.playingBuildingCompleteSound = True
                        pygame.mixer.Channel(5).play(getSound("finish building", self.building.sides), 0)
                        #TODO should reset the building sound too

                    elif self.resources == 0:
                        pygame.mixer.Channel(5).play(getSound("poly armor depleted"),0)

    def startAcceptUpgrade(self):
        self.observe_startAcceptUpgrade(playSound = False)
        for o in self.observers: o.callRemote('startAcceptUpgrade')

    def observe_startAcceptUpgrade(self, playSound = True):
        if playSound:
            pygame.mixer.Channel(5).play(getSound("accept upgrade"))

    def loseResource(self):
        self._loseResource(playSound = False)
        for o in self.observers: o.callRemote('loseResource')
    observe_loseResource = _loseResource

    def _attack(self, playSound=True):
        self.attacking = True
        animation = self.images["Attack"].copy()
        if playSound:
            pygame.mixer.Channel(5).play(getSound("attack"))
        animation.start(6).addCallback(lambda ign: self.events.remove(animation))
        self.events.add(animation)

    def attack(self):
        self._attack(playSound=False)
        for o in self.observers: o.callRemote('attack')

    observe_attack = _attack

    def _updatePosition(self, position, building, playSound=True):
        self.position = position
        # TODO only need this for self.self
        def buildingReset():
            self.building = None
            self._buildingReset = None
#            if hasattr(self.building, 'sides'):# and self.building.sides == 0:# and self.building.resources == 0:
#                print "aborted"
#                self.building.onDestroyed.callback(self.building)

        if playSound:
            if self.scanning.isScanning():
                self.scanFadeOutOk = True
                if not pygame.mixer.Channel(5).get_busy():
                    pygame.mixer.Channel(5).play(getSound("scanning"),-1)
                #else:
                #    pygame.mixer.Channel(5).queue(pygame.mixer.Sound("data/sfx/alex_sfx/Sweeping.wav"))
            else:
                if self.scanFadeOutOk:
                    #pygame.mixer.Channel(5).play(pygame.mixer.Sound("data/sfx/alex_sfx/Sweeping.wav"), -1)
                    pygame.mixer.Channel(5).fadeout(4000)
                    self.scanFadeOutOk = False

        if building:
            self.building = building
            if self._buildingReset and hasattr(self._buildingReset, "cancel"):
                self._buildingReset.cancel()
            self._buildingReset = reactor.callLater(1, buildingReset)

            if (playSound):
                #print('sides : ' + str(hasattr(self.building, 'sides')))
                #print('action: ' + str(self.actionName == 'Building'))
                if (hasattr(self.building, 'sides') and self.actionName == 'Building'):# and self.resources:

                    if self.resources == 0:
                        #if not self.playingBuildingCompleteSound or not pygame.mixer.Channel(7).get_busy():
                        #if (not self.playingBuildingCompleteSound or not pygame.mixer.Channel(5).get_busy()):
                        pygame.mixer.Channel(7).stop()
                        #    self.playingBuildingCompleteSound = False

                    else:
                        buildingSideCount = self.building.sides
                        if buildingSideCount < 3:
                            buildingSideCount = 3
                        else:
                            buildingSideCount = min(buildingSideCount + 1, 5)

                        if (not self.playingBuildingCompleteSound or not pygame.mixer.Channel(5).get_busy()):
                            self.playingBuildingCompleteSound = False
                            if not pygame.mixer.Channel(7).get_busy():
                                pygame.mixer.Channel(7).play(getSound("building", buildingSideCount))
                            else:
                                pygame.mixer.Channel(7).queue(getSound("building", buildingSideCount))

                elif self.actionName == 'Mining':
                    if self.resources < self.sides or self.sides == 0:
                        if not pygame.mixer.Channel(7).get_busy():
                            pygame.mixer.Channel(7).play(getSound("mining"))
                        else:
                            pygame.mixer.Channel(7).queue(getSound("mining"))
                else:
                        pygame.mixer.Channel(7).stop()

    def updatePosition(self, position, building):
        self._updatePosition(position, building, playSound=False)
        for o in self.observers: o.callRemote('updatePosition', position, building)

    observe_updatePosition = _updatePosition

    def _hit(self):
        if self.resources:
            self.setResources(self.resources - 1)
        else:
            animation = self.images["LevelUp"].copy()
            animation.startReversed(72).addCallback(lambda ign: self.topEvents.remove(animation))
            self.topEvents.add(animation)
            self.sides -= 1

    def hit(self):
        self._hit()
        for o in self.observers: o.callRemote('hit')
    observe_hit = _hit

    def _levelUp(self):
        self.armor.clear()
        self.setResources(0)
        self.sides += 1

        #animation = self.images["building upgraded"].copy()
        #animation.start(12).addCallback(lambda ign: self.topEvents.remove(animation))
        #self.topEvents.add(animation)
    def levelUp(self):
        self._levelUp()
        for o in self.observers: o.callRemote('levelUp')
    observe_levelUp = _levelUp

    def paint(self, view, position, isTeammate, isVisible):
        # TODO player image deviates from center of screen occasionally
        # likely caused by view.center being updated but not player.position
        # which must wait for the server to update its
        if self.self:
            (cx, cy) = view.screen.get_rect().center
            position = Vector2D(cx,cy) #Vector2D(240, 400)
        # TODO HACK save the view to get images
        self.images = view.images.images

        if isVisible and self.scanning:
            view.images.images["PlayerScan"].drawScaled(view.screen, position, self.getScanRadius())

        for image in self.events:
            image.draw(view.screen, position)

        if isVisible:
#            if self.miningAnimation == None:
#                self.miningAnimation = self.images["mining"].copy()
#                self.miningAnimation.start(12)
#            self.miningAnimation.draw(view.screen, position)

            image = view.images.images["Player", self.team, self.sides]
            image.draw(view.screen, position)
            for image in self.topEvents:
                image.draw(view.screen, position)
            if self.tooltip:
                self.tooltip.draw(view.screen, position + Vector2D(0, -100))
        else:
            image = view.images.images["Enemy", self.team]
            #image.start(12)
            image.draw(view.screen, position)
            return

        for a in self.armor:
            # XXX Must start all clients at the same time or armor is Unpersistable
            self.armor[a].draw(view.screen, position)

    def getStateToCacheAndObserveFor(self, perspective, observer):
        self.observers.append(observer)
        state = pb.Cacheable.getStateToCopyFor(self, perspective).copy()
        del state['observers']
        if self == perspective.player:
            state['self'] = True
        return state

    def setCopyableState(self, state):
        pb.RemoteCache.setCopyableState(self, state)
        self.scanning = PlayerScan()

    def stoppedObserving(self, perspective, observer):
        self.observers.remove(observer)

pb.setUnjellyableForClass(Player, Player)

class Building(pb.Cacheable, pb.RemoteCache):
    def __init__(self):
        self.sides = 0
        self.resources = 0
        self.observers = []
        self.size = 1
        self.onDestroyed = defer.Deferred()
        self.upgrading = None
        self.explosion = None
        self.upgradeAnim = None

    def build(self, player):
        if not player.resources:
            return
        if self.sides == 5 and self.resources == 5:
            if self.upgrading and self.upgrading.sides > 2:
                player.loseResource()
                if self.upgrading.sides == self.upgrading.resources:
                    self.upgrading.levelUp()
                else:
                    self.upgrading.gainResource()
        else:
            self.gainResource()
            player.loseResource() #have player lose resource after, so they can see if a new building got made.
        for o in player.observers: o.callRemote('setAction', "Building")

    def _gainResource(self, playSound=True):
        # Not a full polyfactory
        # if rubble
        buildingLeveledUp = False
        if not self.sides:
            if self.resources == 2:
                self.sides = 3
                self.resources = 0
                buildingLeveledUp = True
            else:
                self.resources += 1
        else:
            # if armor is full
            if self.sides == self.resources:
                self.sides += 1
                self.resources = 0
                buildingLeveledUp = True
            else:
                self.resources += 1

        if buildingLeveledUp:
            self.upgradeAnim = self.images["building upgraded"].copy()
            self.upgradeAnim.start(12).addCallback(lambda ign: self.clearUpgradeAnim())

    def clearUpgradeAnim(self):
        self.upgradeAnim = None

    def gainResource(self):
        self._gainResource(playSound=False)
        for o in self.observers: o.callRemote('gainResource')

    observe_gainResource = _gainResource

    def observe_setResources(self, r):
        self.resources = r

    def drawToolTip(self, view, tip, team = None):
        # TODO No more tool tips?
        pass

    def paint(self, view, position, isTeammate):
        if self.explosion:
           self.explosion.draw(view.screen, position)
           return

        if self.sides == 0 and self.resources == 0:
            return

        if self.sides >= 3:
            view.images.images["Building Zone", self.sides, self.team].draw(view.screen, position)

        if self.upgradeAnim:
            self.upgradeAnim.draw(view.screen, position)

        if self.sides:
            view.images.images["Building", self.sides, self.team].draw(view.screen, position)
            view.images.images["BuildingHealth", self.team, self.sides, self.resources].draw(view.screen, position)
        else:
            image = view.images.images["Building", self.resources, self.team].draw(view.screen, position)

    def getStateToCacheAndObserveFor(self, perspective, observer):
        self.observers.append(observer)
        state = pb.Cacheable.getStateToCopyFor(self, perspective).copy()
        del state['observers']
        return state

    def stoppedObserving(self, perspective, observer):
        self.observers.remove(observer)

    def hit(self):
        if not (self.sides and self.resources):
            self.onDestroyed.callback(self)
        elif self.resources:
            self.resources -= 1
            for o in self.observers: o.callRemote('setResources', self.resources)

    def _explode(self):
        #TODO a delay to the explosion would be nice.
        self.explosion = self.images["TrapExplosion"].copy()
        return self.explosion.start(4)

    def explode(self):
        self._explode().addCallback(lambda ign: self.onDestroyed.callback(self))
        for o in self.observers: o.callRemote('explode')
    observe_explode = _explode

    def isTrap(self):
        if self.sides == 3 and not self.explosion:
            return True
        return False

    def isSentry(self):
        return self.sides == 4

    def isPolyFactory(self):
        return self.sides == 5

pb.setUnjellyableForClass(Building, Building)

class ResourcePool(pb.Copyable, pb.RemoteCopy):
    def __init__(self, size):
        self.size = 3

    def build(self, player):
        player.gainResource()
        for o in player.observers: o.callRemote('setAction', "Mining")

    def addBuilder(self, player):
        pass

    def removeBuilder(self, player):
        pass

    def drawToolTip(self, view, tip, team):
        # TODO No more tool tips?
        pass

    def paint(self, view, position):
        view.images.images["resource_pool_zone"].draw(view.screen, position)
        view.images.images["resource_pool"].draw(view.screen, position)

pb.setUnjellyableForClass(ResourcePool, ResourcePool)