from datetime import datetime as dt
import matplotlib.pyplot as plt
import numpy as np

string = \
"""size = 5, 2c iteration 0 with desired nach equilibria: [(('row', 'S2'), ('column', 'S1'))] || Time: 0:00:00.089607
size = 5, 2c iteration 1 with desired nach equilibria: [(('row', 'S2'), ('column', 'S1'))] || Time: 0:00:00.008921
size = 5, 2c iteration 2 with desired nach equilibria: [(('row', 'S2'), ('column', 'S1'))] || Time: 0:00:00.005190
size = 5, 2c iteration 3 with desired nach equilibria: [(('row', 'S2'), ('column', 'S1'))] || Time: 0:00:00.005113
size = 5, 2c iteration 4 with desired nach equilibria: [(('row', 'S2'), ('column', 'S1'))] || Time: 0:00:00.004982
size = 10, 2c iteration 0 with desired nach equilibria: [(('row', 'S5'), ('column', 'S5'))] || Time: 0:00:00.002025
size = 10, 2c iteration 1 with desired nach equilibria: [(('row', 'S5'), ('column', 'S5'))] || Time: 0:00:00.004903
size = 10, 2c iteration 2 with desired nach equilibria: [(('row', 'S5'), ('column', 'S5'))] || Time: 0:00:00.010714
size = 10, 2c iteration 3 with desired nach equilibria: [(('row', 'S5'), ('column', 'S5'))] || Time: 0:00:00.005935
size = 10, 2c iteration 4 with desired nach equilibria: [(('row', 'S5'), ('column', 'S5'))] || Time: 0:00:00.005876
size = 20, 2c iteration 0 with desired nach equilibria: [(('row', 'S2'), ('column', 'S13'))] || Time: 0:00:00.002486
size = 20, 2c iteration 1 with desired nach equilibria: [(('row', 'S2'), ('column', 'S13'))] || Time: 0:00:00.010239
size = 20, 2c iteration 2 with desired nach equilibria: [(('row', 'S2'), ('column', 'S13'))] || Time: 0:00:00.007542
size = 20, 2c iteration 3 with desired nach equilibria: [(('row', 'S2'), ('column', 'S13'))] || Time: 0:00:00.008141
size = 20, 2c iteration 4 with desired nach equilibria: [(('row', 'S2'), ('column', 'S13'))] || Time: 0:00:00.006896
size = 30, 2c iteration 0 with desired nach equilibria: [(('row', 'S26'), ('column', 'S18'))] || Time: 0:00:00.051712
size = 30, 2c iteration 1 with desired nach equilibria: [(('row', 'S26'), ('column', 'S18'))] || Time: 0:00:00.022506
size = 30, 2c iteration 2 with desired nach equilibria: [(('row', 'S26'), ('column', 'S18'))] || Time: 0:00:00.010083
size = 30, 2c iteration 3 with desired nach equilibria: [(('row', 'S26'), ('column', 'S18'))] || Time: 0:00:00.011068
size = 30, 2c iteration 4 with desired nach equilibria: [(('row', 'S26'), ('column', 'S18'))] || Time: 0:00:00.015029
size = 50, 2c iteration 0 with desired nach equilibria: [(('row', 'S27'), ('column', 'S7'))] || Time: 0:00:00.052816
size = 50, 2c iteration 1 with desired nach equilibria: [(('row', 'S27'), ('column', 'S7'))] || Time: 0:00:00.299637
size = 50, 2c iteration 2 with desired nach equilibria: [(('row', 'S27'), ('column', 'S7'))] || Time: 0:00:00.047638
size = 50, 2c iteration 3 with desired nach equilibria: [(('row', 'S27'), ('column', 'S7'))] || Time: 0:00:00.055767
size = 50, 2c iteration 4 with desired nach equilibria: [(('row', 'S27'), ('column', 'S7'))] || Time: 0:00:00.064685
size = 100, 2c iteration 0 with desired nach equilibria: [(('row', 'S33'), ('column', 'S88'))] || Time: 0:00:00.181271
size = 100, 2c iteration 1 with desired nach equilibria: [(('row', 'S33'), ('column', 'S88'))] || Time: 0:00:00.164494
size = 100, 2c iteration 2 with desired nach equilibria: [(('row', 'S33'), ('column', 'S88'))] || Time: 0:00:00.156647
size = 100, 2c iteration 3 with desired nach equilibria: [(('row', 'S33'), ('column', 'S88'))] || Time: 0:00:00.169929
size = 100, 2c iteration 4 with desired nach equilibria: [(('row', 'S33'), ('column', 'S88'))] || Time: 0:00:00.215340
size = 250, 2c iteration 0 with desired nach equilibria: [(('row', 'S77'), ('column', 'S149'))] || Time: 0:00:00.689703
size = 250, 2c iteration 1 with desired nach equilibria: [(('row', 'S77'), ('column', 'S149'))] || Time: 0:00:00.852490
size = 250, 2c iteration 2 with desired nach equilibria: [(('row', 'S77'), ('column', 'S149'))] || Time: 0:00:00.910095
size = 250, 2c iteration 3 with desired nach equilibria: [(('row', 'S77'), ('column', 'S149'))] || Time: 0:00:00.947000
size = 250, 2c iteration 4 with desired nach equilibria: [(('row', 'S77'), ('column', 'S149'))] || Time: 0:00:00.914404
size = 400, 2c iteration 0 with desired nach equilibria: [(('row', 'S123'), ('column', 'S332'))] || Time: 0:00:02.228152
size = 400, 2c iteration 1 with desired nach equilibria: [(('row', 'S123'), ('column', 'S332'))] || Time: 0:00:02.548280
size = 400, 2c iteration 2 with desired nach equilibria: [(('row', 'S123'), ('column', 'S332'))] || Time: 0:00:03.533587
size = 400, 2c iteration 3 with desired nach equilibria: [(('row', 'S123'), ('column', 'S332'))] || Time: 0:00:02.408298
size = 400, 2c iteration 4 with desired nach equilibria: [(('row', 'S123'), ('column', 'S332'))] || Time: 0:00:03.090571
size = 700, 2c iteration 0 with desired nach equilibria: [(('row', 'S622'), ('column', 'S495'))] || Time: 0:00:10.831137
size = 700, 2c iteration 1 with desired nach equilibria: [(('row', 'S622'), ('column', 'S495'))] || Time: 0:00:11.929054
size = 700, 2c iteration 2 with desired nach equilibria: [(('row', 'S622'), ('column', 'S495'))] || Time: 0:00:14.781851
size = 700, 2c iteration 3 with desired nach equilibria: [(('row', 'S622'), ('column', 'S495'))] || Time: 0:00:12.353810
size = 700, 2c iteration 4 with desired nach equilibria: [(('row', 'S622'), ('column', 'S495'))] || Time: 0:00:12.368400
size = 1000, 2c iteration 0 with desired nach equilibria: [(('row', 'S441'), ('column', 'S350'))] || Time: 0:00:27.779875
size = 1000, 2c iteration 1 with desired nach equilibria: [(('row', 'S441'), ('column', 'S350'))] || Time: 0:00:24.011225
size = 1000, 2c iteration 2 with desired nach equilibria: [(('row', 'S441'), ('column', 'S350'))] || Time: 0:00:22.359567
size = 1000, 2c iteration 3 with desired nach equilibria: [(('row', 'S441'), ('column', 'S350'))] || Time: 0:00:24.437762
size = 1000, 2c iteration 4 with desired nach equilibria: [(('row', 'S441'), ('column', 'S350'))] || Time: 0:00:36.910838"""

# For each size, we run 5 iterations of the algorithm, and we take the average time
# of the 5 iterations. We then plot the average time against the size of the game.
# We also plot the theoretical time complexity of the algorithm, which is O(n^2).
# We see that the algorithm is indeed O(n^2), as the plot is a straight line.

def get_averages(string):
    sizes = [5, 10, 20, 30, 50, 100, 250, 400, 700, 1000]
    # averages = []
    # for i in range(len(sizes)):
    #     # Get the average time for each size
    #     # Get the lines of the string
    #     lines = string.split('\n')
    #     times = [(line.split('||')[1].split(': ')[1]) for line in lines[i*5:i*5+5]]
    #     # turn the string datetimes into seconds
    #     times = [dt.strptime(time, '%H:%M:%S.%f') for time in times]
    #     times = [time.second + time.microsecond/1000000 for time in times]
    #     print(times)
    #     average = sum(times)/len(times)
    #     averages.append(average)
    # print(averages)
    # return averages

    lines = string_summer_2023.split('\n')

    # Create a dictionary of times for every size.
    times = {}

    for line in lines:
        # Get the size from the line.
        size = line.split('size = ')[1].split(',')[0]

        # Get the time from the line.
        time = dt.strptime(line.split('Time: ')[1], '%H:%M:%S.%f')

        # Format the time as seconds.
        time = time.second + time.microsecond/1000000 + time.minute*60

        print(f"for size {size}, time is {time}")

        # Add the time to the dictionary of times for the given size.
        if size in times:
            times[size].append(time)
        else:
            times[size] = [time]

    # Calculate the average time for every size.
    average_times = {}
    for size in times:
        average_times[size] = sum(times[size])/len(times[size])

    occurences = {size: len(times[size]) for size in times}
    return average_times, occurences

def plot_averages(string, quadratic=True):
    # sizes = [5, 10, 20, 30, 50, 100, 250, 400, 700, 1000]
    # averages = get_averages(string)

    averages_dict, occurences_dict = get_averages(string)
    sizes = [int(size) for size in averages_dict.keys()]
    sizes.sort()
    averages = [averages_dict[str(size)] for size in sizes]

    for size, average in zip(sizes, averages):
        print(f"size {size} has square {size**2} and average {average} over {occurences_dict[str(size)]} occurences")

    # plt.plot(sizes, [size**2 for size in sizes])
    # fit a quadratic curve to the data
    if quadratic:
        fit = np.polyfit(sizes, averages, 2)
        fit_fn = np.poly1d(fit)
        # draw the fit curve fit_fn(sizes) for a range from 0 to 1000
        x = np.linspace(5, max(sizes), 1000)
        # plt.plot(x, fit_fn(x), '--k', label='Quadratic fit', linewidth=1, color='dodgerblue', alpha=0.8)
        plt.plot(x, fit_fn(x), '--k', label='Quadratic fit', linewidth=1.2, color='#00A6B7', alpha=0.8)
        plt.xticks(np.arange(0, max(sizes)+1, 100))
        # give them 45 degree angle
        plt.xticks(rotation=45)

        # plt.plot(sizes, averages, label='Average Time', linewidth=1, color='red')
        # make the color FF6805
        plt.plot(sizes, averages, label='Average Time', linewidth=1.5, color='#FF7F00')
        # label the points with how many times they occured

        # plot the data points
        # plt.scatter(sizes, averages, color='red', s=15)
        plt.scatter(sizes, averages, color='#FF7F00', s=25, zorder=3)
        plt.xlabel('Number of strategies ($n$)', fontsize=15)

        plt.ylabel('Average CPU time (seconds)', fontsize=15)
        plt.title('Average time to change payoffs to new Nash equilibrium')
        # make the y numbers integers and not 1.0e7
        plt.ticklabel_format(style='plain')
        for size, average in zip(sizes, averages):
            plt.text(size, average, occurences_dict[str(size)], fontsize=8, color='black', alpha=0.8)
        # add thin gridlines
        plt.grid(linewidth=0.2)

        # legends
        plt.legend(loc='upper left')

        plt.savefig('large_games_quadratic.png', dpi=300)

        plt.show()

    else: # linear
        sizes = [size**2 for size in sizes]
        fit = np.polyfit(sizes, averages, 1)
        fit_fn = np.poly1d(fit)
        x = np.linspace(25, max(sizes), 1000)
        plt.xticks(np.arange(0, max(sizes)+1, 100000))
        # give them 45 degree angle
        plt.xticks(rotation=45)

        # plt.plot(x, fit_fn(x), '--k', label='Linear fit', linewidth=1, color='dodgerblue', alpha=0.8)
        plt.plot(x, fit_fn(x), '--k', label='Linear fit', linewidth=1.2, color='#00A6B7', alpha=0.8)
        # plt.plot(sizes, averages, label='Average Time', linewidth=1, color='red')
        # make the color FF6805
        plt.plot(sizes, averages, label='Average Time', linewidth=1.5, color='#FF7F00')
        # plt.scatter(sizes, averages, color='red', s=15)
        plt.scatter(sizes, averages, color='#FF7F00', s=25, zorder=3)
        plt.xlabel('Size of game ($n^2$)', fontsize=15)
        for size, average in zip(sizes, averages):
            plt.text(size, average, occurences_dict[str(int(np.sqrt(size)))], fontsize=10, color='black', alpha=0.8, zorder=4)

        # calculate R^2
        residuals = np.polyfit(sizes, averages, 1)
        r_squared = 1 - (np.var(residuals) / np.var(averages))
        r2 = np.corrcoef(sizes, averages)[0, 1]**2
        print(f"R^2 is {r_squared}")
        print(f"R^2 is {r2}")
        # Calculate residuals
        residuals = averages - fit_fn(sizes)

        # Calculate total sum of squares
        total_sum_squares = np.sum((averages - np.mean(averages))**2)

        # Calculate residual sum of squares
        residual_sum_squares = np.sum(residuals**2)

        # Calculate R^2
        r_squared = 1 - (residual_sum_squares / total_sum_squares)

        print("R^2:", r_squared)

        plt.ylabel('Average CPU time (seconds)', fontsize=15)
        plt.title('Average time to change payoffs to new Nash equilibrium')
        # make the y numbers integers and not 1.0e7
        plt.ticklabel_format(style='plain')
        # add thin gridlines
        plt.grid(linewidth=0.2)

        # legends
        plt.legend(loc='upper left')

        plt.savefig('large_games_linear.png', dpi=300)

        plt.show()

# load the file "log Summer 2023.txt" into a string
string_summer_2023 = open('log Summer 2023.txt', 'r').read()
# string_summer_2023 = open('log2.txt', 'r').read()
# string_summer_2023 = open('log Summer 2023 2.txt', 'r').read()
string_summer_2023 = open('log_eristwo.txt', 'r').read()
# plot_averages(string_summer_2023, quadratic=False)
plot_averages(string_summer_2023, quadratic=False)
plot_averages(string_summer_2023, quadratic=True)
# print(get_averages(string_summer_2023))

# 58627   ee869   RUN   normal     eris2n4     hn008       job_740    Jun 29 00:54
# 58632   ee869   PEND  normal     eris2n4                 job_895    Jun 29 00:54
# 58633   ee869   PEND  normal     eris2n4                 job_920    Jun 29 00:54
# 58634   ee869   PEND  normal     eris2n4                 job_950    Jun 29 00:54
# 58635   ee869   PEND  normal     eris2n4                 job_975    Jun 29 00:54
# 61369   ee869   PEND  normal     eris2n4                 job_20     Jun 29 12:22
# 61370   ee869   PEND  normal     eris2n4                 job_315    Jun 29 12:22
# 61371   ee869   PEND  normal     eris2n4                 job_385    Jun 29 12:22
# 61372   ee869   PEND  normal     eris2n4                 job_445    Jun 29 12:22
# 61373   ee869   PEND  normal     eris2n4                 job_500    Jun 29 12:22
# 61374   ee869   PEND  normal     eris2n4                 job_550    Jun 29 12:22
# 61375   ee869   PEND  normal     eris2n4                 job_590    Jun 29 12:22
# 61376   ee869   PEND  normal     eris2n4                 job_630    Jun 29 12:22
# 61377   ee869   PEND  normal     eris2n4                 job_670    Jun 29 12:22
# 61378   ee869   PEND  normal     eris2n4                 job_705    Jun 29 12:22
# 61379   ee869   PEND  normal     eris2n4                 job_740    Jun 29 12:22
# 61380   ee869   PEND  normal     eris2n4                 job_775    Jun 29 12:22
# 61381   ee869   PEND  normal     eris2n4                 job_805    Jun 29 12:22
# 61382   ee869   PEND  normal     eris2n4                 job_835    Jun 29 12:22
# 61383   ee869   PEND  normal     eris2n4                 job_865    Jun 29 12:22
# 61384   ee869   PEND  normal     eris2n4                 job_895    Jun 29 12:22
# 61385   ee869   PEND  normal     eris2n4                 job_920    Jun 29 12:22
# 61386   ee869   PEND  normal     eris2n4                 job_950    Jun 29 12:22
# 61387   ee869   PEND  normal     eris2n4                 job_975    Jun 29 12:22

# bkill 61369
# bkill 61370
# bkill 61371
# bkill 61372
# bkill 61373
# bkill 61374
# bkill 61375
# bkill 61376
# bkill 61377
# bkill 61378
# bkill 61379
# bkill 61380
# bkill 61381
# bkill 61382
# bkill 61383
# bkill 61384
# bkill 61385
# bkill 61386
# bkill 61387


from core.game.Card import Card
from core.game.IPlayer import IPlayer
from core.game.GameEngineHelper import GameEngineHelper
from core.game.PlayerGameHelper import PlayerGameHelper
from core.game.GamePlayer import GamePlayer
from typing import List

INITIAL_HAND_SIZE: int = 7
MIN_REQUIRED_PLAYERS: int = 2
MAX_REQUIRED_PLAYERS: int = 10

class GameEngine:
    def __init__(self, players: List[IPlayer], rounds_per_match: int):
        players_count = len(players)
        self._players: List[IPlayer] = players
        if players_count < MIN_REQUIRED_PLAYERS or players_count > MAX_REQUIRED_PLAYERS:
            raise ValueError('Amount of players must be at least 2 and cannot exceed 10')
        self._roundsPerMatch: int = rounds_per_match
        self._deck: List[Card] = GameEngineHelper.create_game_deck()

    def __get_game_players(self, players: List[IPlayer]) -> List[GamePlayer]:
        game_players = []
        for index, player in enumerate(players):
            player_hand = self.__draw_cards(INITIAL_HAND_SIZE)
            game_player = GamePlayer(player, player_hand, index)
            game_players.append(game_player)
        return game_players

    def __draw_cards(self, draw_count: int) -> List[Card]:
        drawn_cards = []
        for n in range(draw_count):
            drawn_cards.append(self._deck.pop(0))
        return drawn_cards

    def __create_player_game_helper(self, active_player: GamePlayer, last_card_played: Card):
        hand = active_player.hand
        last_card_played = last_card_played
        card_pile = []
        deck_count = len(self._deck)
        opp_hand_count = []
        return PlayerGameHelper(hand, last_card_played, card_pile, deck_count, opp_hand_count)

    def start(self):
        current_round: int = 0

        while current_round <= self._roundsPerMatch:
            # At the beginning of every round, get a fresh list of players and resent round variables.
            round_players = self.__get_game_players(self._players)
            self._deck = GameEngineHelper.create_game_deck()
            current_round += 1
            round_won = False
            round_winner = None
            print('- Round {0}'.format(current_round))

            # Keep playing the same round until a winner is chosen.
            while not round_won:
                active_player: GamePlayer = None
                last_card_played = self.__draw_cards(1)

                # Main game loop starts here, loop through each player until a player has 0 cards.
                for game_player in round_players:
                    active_player = game_player.player
                    active_player_game_helper = self.__create_player_game_helper(game_player, last_card_played)
                    game_player.player.set_game_helper(active_player_game_helper)
                    player_action = game_player.player.take_turn()
                    print('{} uses action {}'.format(active_player.get_player_name(), player_action.action))
                    print('{}'.format(player_action.card.get_card_text()))

                    # Until game loop is actually implemented, remove a card from the player's hand.
                    game_player.hand.pop()

                    # After the card has been played, if that player has no more cards, they win the round.
                    if len(game_player.hand) <= 0:
                        round_won = True
                        print('{} won the round!'.format(active_player.get_player_name()))
                        round_winner = active_player

        # After all rounds have been played, handle any Match over logic here
        print('Match is over!')


from django.test import TestCase
from django.urls import reverse
from ..models import Product

class ProductViewTest(TestCase):
    @classmethod
    def setUpTestData(cls):
        Product.objects.create(name="Test Product", category="cat1", nutrition_grade="e")
        for category in ["cat1", "cat2", "cat3", "cat4"]:
            for grade in ["a", "b", "c", "d", "e"]:
                for e in range(60):  # Creating 60 instances for each different grades
                    Product.objects.create(nutrition_grade=grade, category=category)

    def test_product(self):
        product = Product.objects.all()[0]
        product_url = reverse('product', args=[product.pk])
        response = self.client.get(product_url)
        self.assertEquals(response.context['prod'], Product.objects.get(pk=product.pk))
        self.assertEquals(response.status_code, 200)
        # When launching all tests only : products.models.Product.DoesNotExist: Product matching query does not exist. ;

    def test_search_product(self):
        # Testing with correct input
        response = self.client.post('/products/search', {'research':'Test Product'})
        self.assertEquals(response.status_code, 200)
        # Testing with incorrect input
        response = self.client.post('/products/search', {'research':'WrongInput'})
        self.assertEquals(response.status_code, 302)


"""
Batch-run predictors and recommenders for evaluation.
"""

import logging
import pathlib
import collections
from time import perf_counter
from functools import partial
import warnings

import pandas as pd
import numpy as np

from .algorithms import Predictor, Recommender
from . import topn, util

try:
    import fastparquet
except ImportError:
    fastparquet = None

_logger = logging.getLogger(__name__)

def predict(algo, pairs, model=None):
    """
    Generate predictions for user-item pairs.  The provided algorithm should be a
    :py:class:`algorithms.Predictor` or a function of two arguments: the user ID and
    a list of item IDs. It should return a dictionary or a :py:class:`pandas.Series`
    mapping item IDs to predictions.

    Args:
        predictor(callable or :py:class:algorithms.Predictor):
            a rating predictor function or algorithm.
        pairs(pandas.DataFrame):
            a data frame of (``user``, ``item``) pairs to predict for. If this frame also
            contains a ``rating`` column, it will be included in the result.
        model(any): a model for the algorithm.

    Returns:
        pandas.DataFrame:
            a frame with columns ``user``, ``item``, and ``prediction`` containing
            the prediction results. If ``pairs`` contains a `rating` column, this
            result will also contain a `rating` column.
    """

    if isinstance(algo, Predictor):
        pfun = partial(algo.predict, model)
    else:
        pfun = algo

    def run(user, udf):
        res = pfun(user, udf.item)
        return pd.DataFrame({'user': user, 'item': res.index, 'prediction': res.values})

    ures = (run(user, udf) for (user, udf) in pairs.groupby('user'))
    res = pd.concat(ures)
    if 'rating' in pairs:
        return pairs.join(res.set_index(['user', 'item']), on=('user', 'item'))
    return res

def recommend(algo, model, users, n, candidates, ratings=None):
    """
    Batch-recommend for multiple users.  The provided algorithm should be a
    :py:class:`algorithms.Recommender` or :py:class:`algorithms.Predictor` (which
    will be converted to a top-N recommender).

    Args:
        algo: the algorithm
        model: The algorithm model
        users(array-like): the users to recommend for
        n(int): the number of recommendations to generate (None for unlimited)
        candidates:
            the users' candidate sets. This can be a function, in which case it will
            be passed each user ID; it can also be a dictionary, in which case user
            IDs will be looked up in it.
        ratings(pandas.DataFrame):
            if not ``None``, a data frame of ratings to attach to recommendations when
            available.

    Returns:
        A frame with at least the columns ``user``, ``rank``, and ``item``; possibly also
        ``score``, and any other columns returned by the recommender.
    """

    if isinstance(candidates, dict):
        candidates = candidates.get
    algo = Recommender.adapt(algo)

    results = []
    for user in users:
        _logger.debug('generating recommendations for %s', user)
        ucand = candidates(user)
        res = algo.recommend(model, user, n, ucand)
        iddf = pd.DataFrame({'user': user, 'rank': np.arange(1, len(res) + 1)})
        results.append(pd.concat([iddf, res], axis='columns'))

    results = pd.concat(results, ignore_index=True)
    if ratings is not None:
        # combine with test ratings for relevance data
        results = pd.merge(results, ratings, how='left', on=('user', 'item'))
        # fill in missing 0s
        results.loc[results.rating.isna(), 'rating'] = 0

    return results

_AlgoRec = collections.namedtuple('_AlgoRec', [
    'algorithm',
    'parallel',
    'attributes'
])
_DSRec = collections.namedtuple('_DSRec', [
    'dataset',
    'candidates',
    'attributes'
])

class MultiEval:
    """
    A runner for carrying out multiple evaluations, such as parameter sweeps.

    Args:
        path(str or :py:class:`pathlib.Path`):
            the working directory for this evaluation.
            It will be created if it does not exist.
        nrecs(int):
            the number of recommendations to generate per user (None to disable top-N).
        candidates:
            the default candidate set generator.
    """

    def __init__(self, path, nrecs=100, candidates=topn.UnratedCandidates):
        self.workdir = pathlib.Path(path)
        self.n_recs = nrecs
        self.candidate_generator = candidates
        self.algorithms = []
        self.datasets = []

    @property
    def run_csv(self):
        return self.workdir / 'runs.csv'

    @property
    def run_file(self):
        return self.workdir / 'runs.parquet'

    @property
    def preds_file(self):
        return self.workdir / 'predictions.parquet'

    @property
    def recs_file(self):
        return self.workdir / 'recommendations.parquet'

    def add_algorithms(self, algos, parallel=False, attrs=[], **kwargs):
        """
        Add one or more algorithms to the run.

        Args:
            algos(algorithm or list): the algorithm(s) to add.
            parallel(bool):
                if ``True``, allow this algorithm to be trained in parallel with others.
            attrs(list of str):
                a list of attributes to extract from the algorithm objects and include in
                the run descriptions.
            kwargs:
                additional attributes to include in the run descriptions.
        """

        if not isinstance(algos, collections.Iterable):
            algos = [algos]

        for algo in algos:
            aa = {'AlgoClass': algo.__class__.__name__, 'AlgoStr': str(algo)}
            aa.update(kwargs)
            for an in attrs:
                aa[an] = getattr(algo, an, None)

            self.algorithms.append(_AlgoRec(algo, parallel, aa))

    def add_datasets(self, data, name=None, candidates=None, **kwargs):
        """
        Add one or more datasets to the run.

        Args:
            data:
                the input data set(s) to run. Can be one of the followin:

                * A tuple of (train, test) data.
                * An iterable of (train, test) pairs, in which case the iterable
                  is not consumed until it is needed.
                * A function yielding either of the above, to defer data load
                  until it is needed.

            kwargs:
                additional attributes pertaining to these data sets.
        """

        attrs = {}
        if name is not None:
            attrs['DataSet'] = name
        attrs.update(kwargs)

        self.datasets.append(_DSRec(data, candidates, attrs))

    def run(self):
        """
        Run the evaluation.
        """

        self.workdir.mkdir(parents=True, exist_ok=True)

        run_id = 0
        run_data = []

        for ds, cand_f, ds_attrs in self.datasets:
            # normalize data set to be an iterable of tuples
            if callable(ds):
                ds = ds()
            if isinstance(ds, tuple):
                ds = [ds]
            if cand_f is None:
                cand_f = self.candidate_generator

            # loop the data
            for part, (train, test) in enumerate(ds):
                dsp_attrs = dict(ds_attrs)
                dsp_attrs['Partition'] = part + 1
                ds_name = ds_attrs.get('DataSet', None)
                cand = cand_f(train)

                for arec in self.algorithms:
                    run_id += 1

                    _logger.info('starting run %d: %s on %s:%d', run_id, arec.algorithm,
                                 ds_name, part + 1)
                    run = self._run_algo(run_id, arec, (train, test, dsp_attrs, cand))
                    _logger.info('finished run %d: %s on %s:%d', run_id, arec.algorithm,
                                 ds_name, part + 1)
                    run_data.append(run)
                    run_df = pd.DataFrame(run_data)
                    # overwrite files to show progress
                    run_df.to_csv(self.run_csv)
                    run_df.to_parquet(self.run_file, compression=None)

    def _run_algo(self, run_id, arec, data):
        train, test, dsp_attrs, cand = data

        run = {'RunId': run_id}
        run.update(dsp_attrs)
        run.update(arec.attributes)

        model, train_time = self._train_algo(arec.algorithm, train)
        run['TrainTime'] = train_time

        preds, pred_time = self._predict(run_id, arec.algorithm, model, test)
        run['PredTime'] = pred_time
        self._write_results(self.preds_file, preds, append=run_id > 1)

        recs, rec_time = self._recommend(run_id, arec.algorithm, model, test, cand)
        run['recTime'] = rec_time
        self._write_results(self.recs_file, recs, append=run_id > 1)

        return run

    def _train_algo(self, algo, train):
        watch = util.Stopwatch()
        _logger.info('training algorithm %s on %d ratings', algo, len(train))
        model = algo.train(train)
        watch.stop()
        _logger.info('trained algorithm %s in %s', algo, watch)
        return model, watch.elapsed()

    def _predict(self, rid, algo, model, test):
        if not isinstance(algo, Predictor):
            return None, None

        watch = util.Stopwatch()
        _logger.info('generating %d predictions for %s', len(test), algo)
        preds = predict(algo, test, model)
        watch.stop()
        _logger.info('generated predictions in %s', watch)
        preds['RunId'] = rid
        preds = preds[['RunId', 'user', 'item', 'rating', 'prediction']]
        return preds, watch.elapsed()

    def _recommend(self, rid, algo, model, test, candidates):
        if self.n_recs is None:
            return None, None

        watch = util.Stopwatch()
        users = test.user.unique()
        _logger.info('generating recommendations for %d users for %s', len(users), algo)
        recs = recommend(algo, model, users, self.n_recs, candidates, test)
        watch.stop()
        _logger.info('generated recommendations in %s', watch)
        recs['RunId'] = rid
        return recs, watch.elapsed()

    def _write_results(self, file, df, append=True):
        if df is None:
            return

        if fastparquet is not None:
            fastparquet.write(str(file), df, append=append, compression='snappy')
        elif append and file.exists():
            warnings.warn('fastparquet not available, appending is slow')
            odf = pd.read_parquet(str(file))
            pd.concat([odf, df]).to_parquet(str(file))
        else:
            df.to_parquet(str(file))


from django.contrib.auth.decorators import login_required
from django.shortcuts import render, redirect
from django.contrib.auth.models import User
from django.contrib import messages
from django.urls import reverse

from questions.models import QuestionsModel
from cheaters.models import CheatersModel
from gossips.models import GossipsModel
from answers.models import AnswersModel

from .forms import UserUpdateForm, ProfileUpdateForm

@login_required()
def user_profile(request):
    email = request.user.email
    if request.method == 'POST':
        u_form = UserUpdateForm(request.POST, instance=request.user)
        p_form = ProfileUpdateForm(request.POST, request.FILES, instance=request.user.profile)
        if u_form.is_valid() and p_form.is_valid():
            user = u_form.save(commit=False)
            user.email = email
            user.save()
            # u_form.save()
            p_form.save()
            messages.success(request, 'Profile updated successfully')
        else:
            messages.warning(request, 'Some errors occured while updating your profile')
            return render(request, 'users/profile.html', context={'u_form': u_form, 'p_form': p_form})
        return redirect('users:user_profile')

    p_form = ProfileUpdateForm(instance=request.user.profile)
    u_form = UserUpdateForm(instance=request.user)
    user_profile = User.objects.get(username=request.user.username)

    context = {'p_form': p_form, 'u_form': u_form, 'email': email, 'user_profile': user_profile}
    return render(request, 'users/profile.html', context)

def user_view_profile(request, username):
    try:
        user_profile = User.objects.get(username=username)
    except:
        messages.warning(request, "An error occured while trying to get the user details")
        return redirect('gossips:gossips_index')

    user_questions = QuestionsModel.objects.filter(author=user_profile).order_by('-date_published')
    user_gossips = GossipsModel.objects.filter(author=user_profile).order_by('-date_published')
    user_cheaters = CheatersModel.objects.filter(author=user_profile).order_by('-date_published')
    user_answers = AnswersModel.objects.filter(author__username=username)

    followers = user_profile.profile.followers.all()
    following = user_profile.profile.following.all()

    context = {'user_profile': user_profile,
                'user_questions': user_questions,
                'user_gossips': user_gossips,
                'user_cheaters': user_cheaters,
                'user_answers' : user_answers,
                'followers': followers,
                'following': following}
    return render(request, 'users/view_profile.html', context)

@login_required()
def follow_user(request, username):
    logged_in_user = request.user

    try:
        user_to_follow = User.objects.get(username=username)
    except:
        messages.warning(request, "Error! Unable to follow that user")
        return redirect('gossips:gossips_index')

    if logged_in_user == user_to_follow:
        messages.warning(request, "You cannot follow yourself!")
        return redirect('gossips:gossips_index')
    else:
        if logged_in_user in user_to_follow.profile.followers.all():
            user_to_follow.profile.followers.remove(logged_in_user)
            user_to_follow.save()

            logged_in_user.profile.following.remove(user_to_follow)
            logged_in_user.save()
            # messages.success(request, f"You unfollowed {user_to_follow.username}")
        else:
            user_to_follow.profile.followers.add(logged_in_user)
            user_to_follow.save()

            logged_in_user.profile.following.add(user_to_follow)
            logged_in_user.save()
            # messages.success(request, f"You followed {user_to_follow.username}")

    return redirect(reverse('users:user_view_profile', kwargs={'username': user_to_follow.username}))


import argparse
import glob
import json
import os
import shutil
from subprocess import check_call

install_schemes = {
    'system': {
        'application': '/opt',
        'commands': '/usr/local/bin',
        'desktop': '/usr/local/share/applications',
        'icons': '/usr/local/share/icons',
        'mimetypes': '/usr/local/share/mime/packages',
    },
    'user': {
        'application': '{XDG_DATA_HOME}/installed-applications',
        'commands': '~/.local/bin',
        'desktop': '{XDG_DATA_HOME}/applications',
        'icons': '{XDG_DATA_HOME}/icons',
        'mimetypes': '{XDG_DATA_HOME}/mime/packages',
    }
}

class ApplicationInstaller(object):
    def __init__(self, directory, scheme):
        self.directory = directory
        self.scheme = install_schemes[scheme]
        with open(self._relative('batis_info', 'metadata.json')) as f:
            self.metadata = json.load(f)

    def _relative(self, *path):
        return os.path.join(self.directory, *path)

    def destination(self, component):
        XDG_DATA_HOME = os.environ.get('XDG_DATA_HOME') or "~/.local/share"
        path = os.path.expanduser(self.scheme[component].format(XDG_DATA_HOME=XDG_DATA_HOME))
        os.makedirs(path, mode=0o755)
        return path

    def copy_application(self):
        shutil.copytree(self.directory, os.path.join(self.destination('application'),
                                                     os.name))

    def install_commands(self):
        pass

    def install_icons(self):
        pass
        # check_call(['xdg-icon-resource', 'install', '--noupdate', '--theme', theme,
        #       '--size', size, '--context', context, file])
        # check_call(['xdg-icon-resource', 'forceupdate', '--theme', theme])

    def install_mimetypes(self):
        for file in glob.glob(self._relative('batis_info', 'mime', '*.xml')):
            check_call(['xdg-mime', 'install', file])

    def install_desktop_files(self):
        files = glob.glob(self._relative('batis_info', 'desktop', '*.desktop'))
        check_call(['desktop-file-install', '--rebuild-mime-info-cache'] + files)

    def install(self):
        self.copy_application()
        self.install_commands()
        self.install_icons()
        self.install_mimetypes()
        self.install_desktop_files()

def main(argv=None):
    ap = argparse.ArgumentParser()
    ap.add_argument('--system', action='store_true',
            help='Install systemwide, instead of for the user')
    ap.add_argument('directory', help='The unpacked application tarball')
    args = ap.parse_args(argv)
    ai = ApplicationInstaller(args.directory, 'system' if args.system else 'user')
    ai.install()

if __name__ == '__main__':
    main()


import csv
import cv2
import numpy as np

from keras.models import Sequential
from keras.layers import Flatten, Dense, Lambda, Cropping2D
from keras.layers.convolutional import Convolution2D
from keras.layers.pooling import MaxPooling2D

lines = []
with open('./data/driving_log.csv') as csv_file:
    csv_reader = csv.reader(csv_file)
    next(csv_reader) # to skip header row in original data csv
    for line in csv_reader:
        lines.append(line)

correction      = 0.2 # this is a parameter to tune
top_crop        = 70
bot_crop        = 25
img_path        = './data/IMG/'
images          = []
measurements    = []
for line in lines:
    file_name = line[0].split('/')[-1] # centre img
    images.append(cv2.imread(img_path + file_name))
    file_name = line[1].split('/')[-1] # left img
    images.append(cv2.imread(img_path + file_name))
    file_name = line[2].split('/')[-1] # right img
    images.append(cv2.imread(img_path + file_name))

    steering_center = float(line[3])
    # create adjusted steering measurements for the side camera images
    steering_left = steering_center + correction
    steering_right = steering_center - correction

    measurements.extend([steering_center, steering_left, steering_right])

augmented_images, augmented_measurements = [], []
for image, measurement in zip(images, measurements):
    augmented_images.append(image)
    augmented_measurements.append(measurement)
    augmented_images.append(cv2.flip(image, 1))
    augmented_measurements.append(measurement*-1.0)

X_train = np.array(augmented_images)
y_train = np.array(augmented_measurements)

row, col, ch = 160, 320, 3  # Raw image format
# row, col, ch = 160 - top_crop - bot_crop, 320, 3  # Trimmed image format

model = Sequential()
# model.add(Lambda(lambda x: (x/255.0) - 0.5, input_shape=(row,col,ch)))
# model.add(Cropping2D(cropping=((top_crop,bot_crop),(0,0)), input_shape=(row,col,ch)))
model.add(Lambda(lambda x: (x/127.5) - 1., input_shape=(row,col,ch), output_shape=(row,col,ch)))
model.add(Cropping2D(cropping=((top_crop,bot_crop),(0,0)), input_shape=(row,col,ch)))
model.add(Convolution2D(24,5,5, subsample=(2,2), activation="relu"))
model.add(Convolution2D(36,5,5, subsample=(2,2), activation="relu"))
model.add(Convolution2D(48,5,5, subsample=(2,2), activation="relu"))
model.add(Convolution2D(64,3,3, activation="relu"))
model.add(Convolution2D(64,3,3, activation="relu"))
model.add(Flatten())
model.add(Dense(100))
model.add(Dense(50))
model.add(Dense(10))
model.add(Dense(1))

model.compile(loss='mse', optimizer='adam')
model.fit(X_train, y_train, validation_split=0.2, shuffle=True, nb_epoch=3)

model.save('model_nv.h5')


"""
This spider is a JobtoolsAU spider created on top of the ATSSpider
scrapy crawl jobtools_au -a mining_job_id=9999 -a iteration=1 -a extract=1 -a url="https://smartjobs.qld.gov.au/jobtools/jncustomsearch.searchResults?in_organid=14904"

sample url:
https://smartjobs.qld.gov.au/jobtools/jncustomsearch.searchResults?in_organid=14904
"""

from re import compile
from scrapy.http import Request, FormRequest
from scrapy.selector import Selector
from urlparse import urljoin, urlparse

from brightcorp.base.atsspiders import ATSSpider
from brightcorp.items import BrightcorpItemLoader
from brightcorp.processors import Prefix, ConvertDateString, Replace

class JobtoolsAU(ATSSpider):

    name = "jobtools_au"
    ref_re = compile(r"in_jnCounter=(\d+)")
    comp_re = compile(r"\('(.*)'\)")
    next_url_anchor = 'jncustomsearch.searchAction'
    hidden_name = 'in_storeNextBut'
    hidden_value = 'Next'
    logo_url = ''
    jobtype_xpath = None
    exp_date_format = None
    post_date_format = '%d-%m-%Y'
    headers = {}
    desc_xpath = [
        '//div[@id="overview"]',
        '//div[@id="further-information"]',
    ]
    alt_desc_xpaths = []

    def parse(self, response):
        sel = Selector(response)
        if not self.expected_job_count_set:
            job_count = sel.xpath(
                '//h2[@class="resultset-title"]/strong[2]/text()').extract()
            if job_count:
                self.expected_job_count = job_count

        jobs = sel.xpath('//ol[@class="search-results jobs"]/li')
        for job in jobs:
            job_url = job.xpath('./h3/a/@href').extract()
            if job_url:
                job_url = urljoin(response.url, job_url[0])
                meta = {
                    'title': job.xpath(
                        './/span[@class="result-title"]/strong/text()'
                        ).extract(),
                    'jobtype': job.xpath(
                        './/span[@class="type"]/text()'
                        ).extract(),
                    'loc': job.xpath(
                        './/strong[@class="locality"]/text()'
                        ).extract(),
                    'sal': job.xpath(
                        './/span[@class="salary"]/text()'
                        ).extract(),
                }
                yield Request(
                    job_url, callback=self.parse_job_callback(), meta=meta
                )

        if jobs:
            for i in self.parse_next_page(response):
                yield i

    def parse_job(self, response):
        loader = BrightcorpItemLoader(response=response)

        loader.add_value('url', response.url)
        loader.add_value('logo_url', self.logo_url)
        loader.add_value('title', response.meta.get('title'))
        loader.add_value('baseSalary', response.meta.get('sal'))
        loader.add_value(
            'location', response.meta.get('loc'), Replace('Location -')
            )
        loader.add_value(
            'jobcategory', response.meta.get('cat'), Replace('Category -')
            )
        loader.add_value(
            'referencenumber', response.url,
            Prefix('%s-' % self.name), re=self.ref_re
            )
        loader.add_value(
            'date', response.meta.get('date'),
            ConvertDateString(self.post_date_format)
            )

        if self.exp_date_format:
            loader.add_value(
                'expiration_date', response.meta.get('exp_date'),
                ConvertDateString(self.exp_date_format)
            )
        else:
            loader.add_xpath(
                'expiration_date',
                '//th[text()="Closing date"]/following-sibling::td/text()',
                ConvertDateString('%d-%b-%Y')
            )

        loader.add_value('jobtype', response.meta.get('jobtype'))
        if not loader.get_output_value('jobtype'):
            loader.add_xpath('jobtype', self.jobtype_xpath)

        loader.add_xpath(
            'company',
            '//script[contains(text(), "writeAgency(")]/text()',
            re=self.comp_re
        )
        loader.add_xpath(
            'jobcategory',
            '//th[text()="Occupational group"]/following-sibling::td/text()'
        )
        loader.add_xpath(
            'duration',
            '//th[text()="Job duration"]/following-sibling::td/text()'
        )
        loader.add_xpath('description', self.desc_xpath)
        if not loader.get_output_value('description'):
            loader.add_xpath('description', self.alt_desc_xpaths)

        yield loader.load_item()

    def get_form_hidden_inputs(self, sel):
        formdata = {}
        inputs = sel.xpath('//form[@name="resultsform"]/input')
        for _input in inputs:
            formdata[''.join(_input.xpath('./@name').extract())] = ''.join(_input.xpath('./@value').extract())

        return formdata

    def parse_next_page(self, response):
        sel = Selector(response)
        form_data = self.get_form_hidden_inputs(sel)
        form_data[self.hidden_name] = self.hidden_value
        yield FormRequest(
            urljoin(response.url, self.next_url_anchor),
            formdata=form_data, callback=self.parse, headers=self.headers
        )


#!/usr/bin/python3

#Daemon manager for dnsserverp.py. Use this to run dns server as daemon, or use
#initscripts or systemd
#Run this manager with arguments [-start] or [-stop]
#
#Written for python3.4
#
#
#
#Forking daemon based off
#  http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/
#

import sys, os, signal
import argparse
import time
import pwd

import dumbdns

##### Settings (change these): #####

#PID file location. The dumb-dns-server's process id will be stored here so we can
#find the process again when we want to kill the server
PID_FILE = "/var/run/dumbdns-server.pid"

def main():
   #Check if root user, if not, then exit. We need root privilege to write pid
   #files to /var/run
   if(os.getuid() != 0):
      print("You need to be root to run this")
      sys.exit(0)

   #Read command line arguments
   parser = argparse.ArgumentParser(description='dnsserver: a simple DNS server in Python3.')
   group = parser.add_mutually_exclusive_group()
   group.add_argument('-start', action='store_true', help="Start dumb-dns")
   group.add_argument('-stop', action='store_true', help="Stop dumb-dns")

   args = parser.parse_args()
   if not (args.start or args.stop):
      parser.error("Please select -start or -stop")

   #Try to lookup any previous saved process IDs
   try:
      with open(PID_FILE) as f:
         pid = int(f.readline().strip())
         if(isinstance(pid, int)):
            if(args.start):
               print("Existing pid file at {} with process ID {}".format(PID_FILE, pid))
               print("You should run -stop before trying to start a new daemon")
               sys.exit(1)
            elif(args.stop):
               stopDaemon(pid)

   #PID_FILE was not found
   except FileNotFoundError:
      if(args.start):
         startDaemon()
      elif(args.stop):
         print("Could not find pid file at {}".format(PID_FILE))
         print("Lost track of daemon, if it's running you should manually terminate it")
         sys.exit(1)

def startDaemon():

   #Begin forking daemon process, double fork used to completely decouple the daemon
   #Double forking is normally overkill and uneccesary
   #  read the following for more info:
   #  http://stackoverflow.com/questions/881388/
   try:
      pid = os.fork()
      if(pid > 0):
         #exit first parent
         sys.exit(0)
   except OSError as e:
      sys.stderr.write(e.strerror)
      sys.exit(2)

   #decouple from parent environment
   os.chdir("/")
   os.setsid()
   os.umask(0)

   #Second fork
   try:
      pid = os.fork()
      if(pid > 0):
         #exit second parent
         sys.exit(0)
   except OSError as e:
      sys.stderr.write(e.strerror)
      sys.exit(2)

   #write pid to pidfile
   pid = str(os.getpid())
   try:
      with open(PID_FILE, "w") as f:
         f.write("{}\n".format(pid))
   except:
      sys.exit(2)

   #redirect standard file descriptors to null, everything after this should go to
   #a log file
   sys.stdout.flush()
   sys.stderr.flush()
   sin = open(os.devnull, "r")
   sout = open(os.devnull, "a+")
   serr = open(os.devnull, "a+")
   os.dup2(sin.fileno(), sys.stdin.fileno())
   os.dup2(sout.fileno(), sys.stdout.fileno())
   os.dup2(serr.fileno(), sys.stderr.fileno())

   #Run daemon (dumbdns)
   dumbdns.startServer()

def stopDaemon(pid):
   print("Stopping daemon (pid = {})".format(pid))
   os.remove(PID_FILE)
   try:
      os.kill(pid, signal.SIGTERM)
   except:
      pass

if __name__ == '__main__':
   main()


import json
import pickle
import argparse
import numpy as np
from collections import namedtuple

Turn = namedtuple('Turn', ['in_edge', 'outflows', 'probabilities'])
Outflow = namedtuple('Outflow', ['out_edge', 'phase'])

def get_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--turns-file', required=True, help='Turns file (.pkl)')
    parser.add_argument('--flow-file', required=True, help='Flow file (.json)')
    parser.add_argument('--output-file', required=True, help='Output file')
    parser.add_argument('--sample-count', type=int, required=True, help='Sample count')
    args = parser.parse_args()
    return args

def sample_outflows_for_edge(edge, turns, sample_count):
    outflow_count = len(turns.outflows)
    indices = np.random.choice(outflow_count,
                               size=sample_count,
                               p=turns.probabilities)
    return [turns.outflows[index] for index in indices]

# Warning: Works only for a single intersection!
def main():
    args = get_args()

    with open(args.turns_file) as fp:
        turns = pickle.load(fp)

    with open(args.flow_file) as fp:
        flows = json.load(fp)

    samples = {}

    for flow in flows:

        route = flow['route'].split()
        count = flow['count']
        prefix = flow['id']

        for vehIndex in range(count):
            vehID = '{}.{}'.format(prefix, vehIndex)
            samples[vehID] = {}

            processed_edges = set()
            unprocessed_edges = set()

            for edge in route:
                if edge in unprocessed_edges:
                    raise ValueError('{} occurs twice in {}\'s route'.format(edge, vehID))
                unprocessed_edges.add(edge)

            while unprocessed_edges:

                edge = unprocessed_edges.pop()

                # Don't sample on boundary edges
                if edge not in turns:
                    continue

                if edge in processed_edges:
                    continue

                sampled_outflows = sample_outflows_for_edge(edge, turns[edge], args.sample_count)
                samples[vehID][edge] = sampled_outflows
                processed_edges.add(edge)

                for (out_edge, phase) in sampled_outflows:
                    if out_edge not in processed_edges:
                        unprocessed_edges.add(out_edge)

    with open(args.output_file, 'w') as fp:
        pickle.dump(samples, fp)

if __name__ == '__main__':
    main()


'''
This script is heavily inspired by the `run_glue.py` script provided by the HuggingFace team here:
https://github.com/huggingface/transformers/blob/master/examples/run_glue.py
'''
import sys

sys.path.append('../reformer_pytorch')

import argparse
import glob
import json
import logging
import os
import random
from datetime import datetime
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset
from torch.utils.data.distributed import DistributedSampler
from torch.utils.tensorboard import SummaryWriter
from tqdm import tqdm, trange
from reformer_pytorch import Reformer
from reformer_lm_pytorch import ReformerLM

from transformers import BertTokenizer, AdamW, get_linear_schedule_with_warmup
from transformers.data.metrics import glue_compute_metrics as compute_metrics
from transformers.data.processors.glue import glue_convert_examples_to_features as convert_examples_to_features
from transformers.data.processors.glue import glue_output_modes as output_modes
from transformers.data.processors.glue import glue_processors as processors

class FCSoftmax(nn.Module):
    def __init__(self, input_dim, output_dim):
        super(FCSoftmax, self).__init__()
        self.fc = nn.Linear(input_dim, output_dim)
        self.softmax = nn.Softmax()

    def forward(self, x):
        x = self.fc(x)
        x = self.softmax(x)
        return x

class TrainerGLUE(object):

    def __init__(self,
                 model,
                 tokenizer,
                 max_seq_len=8192,
                 task=None,
                 output_mode=None,
                 seed=42,
                 model_name_or_path=None,
                 per_gpu_train_batch_size=8,
                 per_gpu_eval_batch_size=8,
                 output_dir='./output_data',
                 data_dir='../data/glue',
                 fp16=False,
                 fp16_opt_level=0,
                 num_tokens = None,
                 output_dim = 2):
        self.device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
        self.seed = seed
        self.model_name_or_path = model_name_or_path

        self.fc_layer = FCSoftmax(input_dim = num_tokens, output_dim = output_dim).to(self.device)
        self.model = nn.Sequential(model, self.fc_layer).to(self.device)

        self.tokenizer = tokenizer
        self.max_seq_len = max_seq_len
        self.n_gpu = torch.cuda.device_count() if torch.cuda.is_available() else 0

        self.per_gpu_train_batch_size = per_gpu_train_batch_size
        self.train_batch_size = per_gpu_train_batch_size * max(1, self.n_gpu)

        self.per_gpu_eval_batch_size = per_gpu_eval_batch_size
        self.eval_batch_size = self.per_gpu_eval_batch_size * max(1, self.n_gpu)

        self.fp16 = fp16
        self.fp16_opt_level = fp16_opt_level

        self.data_dir = str(data_dir) + '/' + str(task)
        self.output_dir = output_dir

        self.task = task
        self.output_mode = output_modes[self.task]

        self.loss_fn = nn.CrossEntropyLoss()
        self.writer = SummaryWriter()

    def set_seed(self):
        random.seed(self.seed)
        np.random.seed(self.seed)
        torch.manual_seed(self.seed)
        if self.n_gpu > 0:
            torch.cuda.manual_seed_all(self.seed)

    def train(self,
              train_dataset,
              task_name,
              num_train_epochs=3.0,
              weight_decay=0.0,
              learning_rate=5e-5,
              adam_epsilon=1e-8,
              warmup_steps=0,
              max_grad_norm=1.0,
              max_steps=-1,
              gradient_accumulation_steps=1,
              logging_steps=1000,
              save_steps=10000,
              ):

        train_sampler = RandomSampler(train_dataset)
        train_dataloader = DataLoader(train_dataset, batch_size=self.train_batch_size, sampler=train_sampler)

        if max_steps > 0:
            t_total = max_steps
            num_train_epochs = max_steps // (len(train_dataloader) // gradient_accumulation_steps) + 1
        else:
            t_total = len(train_dataloader) // gradient_accumulation_steps * num_train_epochs

        no_decay = ['bias', 'LayerNorm.weight']
        optimizer_grouped_parameters = [
            {
                'params': [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay)],
                'weight_decay': weight_decay
            },
            {
                'params': [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay)],
                'weight_decay': 0.0
            }
        ]

        optimizer = AdamW(optimizer_grouped_parameters, lr=learning_rate, eps=adam_epsilon)
        scheduler = get_linear_schedule_with_warmup(
            optimizer,
            num_warmup_steps=warmup_steps,
            num_training_steps=t_total
        )

        # checking if another optimizer/scheduler exists
        if os.path.isfile('optimizer.pt') and os.path.isfile('scheduler.pt'):
            # if so, load them
            optimizer.load_state_dict(torch.load('optimizer.pt'))
            scheduler.load_state_dict(torch.load('scheduler.pt'))

        if self.fp16:
            try:
                from apex import amp
            except ImportError:
                raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.")
            self.model, optimizer = amp.initialize(self.model, optimizer, opt_level=self.fp16_opt_level)

        if self.n_gpu > 1:
            self.model = torch.nn.DataParallel(self.model)

        # Training

        global_step = 0
        epochs_trained = 0
        steps_trained_in_current_epoch = 0

        # check to see if we are continuing from a checkpoint
        if self.model_name_or_path is not None:
            if os.path.exists(self.model_name_or_path):
                global_step = int(self.model_name_or_path.split('-')[-1].split('/')[0])
                epochs_trained = global_step // (len(train_dataloader) // gradient_accumulation_steps)
                steps_trained_in_current_epoch = global_step % (len(train_dataloader) // gradient_accumulation_steps)

        tr_loss, logging_loss = 0.0, 0.0
        self.model.zero_grad()
        train_iterator = trange(
            epochs_trained, int(num_train_epochs), desc='Epoch', disable=False
        )
        self.set_seed()

        epoch_num = 1

        for _ in train_iterator:
            epoch_iterator = tqdm(train_dataloader, desc='Iteration', disable=False)

            for step, batch in enumerate(epoch_iterator):

                # skip any steps already trained on if picking up from a checkpoint
                if steps_trained_in_current_epoch > 0:
                    steps_trained_in_current_epoch -= 1
                    continue

                self.model.train()
                batch = tuple(t.to(self.device) for t in batch)

                input_ids = batch[0]
                attention_mask = batch[1]
                token_type_ids = batch[2]
                labels = torch.tensor(batch[3], dtype=torch.long, device = self.device)

                # probably only need to pass the input_ids, after we
                # apply the masking ourselves
                outputs = self.model(input_ids)
                outputs = torch.argmax(outputs, dim=-1)

                loss = self.loss_fn(outputs.float(), labels.long())
                loss.requires_grad = True

                if self.n_gpu > 1:
                    loss = loss.mean()
                if gradient_accumulation_steps > 1:
                    loss = loss / gradient_accumulation_steps

                if self.fp16:
                    with amp.scale_loss(loss, optimizer) as scaled_loss:
                        scaled_loss.backward()
                else:
                    loss.backward()

                tr_loss += loss.item()

                epoch_iterator.set_description('Iteration %d: [loss = %f]' % (step, loss.item()))
                train_iterator.set_description('Epoch %d: [avg_loss = %f]' % (epoch_num, tr_loss / float(step + 1)))

                # tensorboard logging
                self.writer.add_scalar('loss/train', loss.item(), global_step)

                if (step + 1) % gradient_accumulation_steps == 0:
                    if self.fp16:
                        torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), max_grad_norm)
                    else:
                        torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_grad_norm)

                    optimizer.step()
                    scheduler.step()
                    self.model.zero_grad()
                    global_step += 1

                    if logging_steps > 0 and global_step % logging_steps == 0:
                        logs = {}

                        results = self.evaluate(task_name)
                        for key, value in results.items():
                            eval_key = f'eval_{key}'
                            logs[eval_key] = value
                            self.writer.add_scalar(str(key) + '/eval', value, global_step)

                        loss_scalar = (tr_loss - logging_loss) / logging_steps
                        learning_rate_scalar = scheduler.get_lr()[0]
                        logs['learning_rate'] = learning_rate_scalar
                        logs['loss'] = loss_scalar
                        logging_loss = tr_loss

                        for key, value in logs.items():
                            # add logging
                            pass
                        print(json.dumps({**logs, **{"step": global_step}}))

                if save_steps > 0 and global_step % save_steps == 0:
                    output_dir = os.path.join(self.output_dir, f'checkpoint-{global_step}')
                    if not os.path.exists(output_dir):
                        os.makedirs(output_dir)
                    model_to_save = (
                        self.model.module if hasattr(self.model, 'module') else self.model
                    )
                    torch.save(model_to_save, f'{output_dir}.pt')
                    torch.save(optimizer.state_dict(), os.path.join(output_dir, 'optimizer.pt'))
                    torch.save(scheduler.state_dict(), os.path.join(output_dir, 'scheduler.pt'))

                if 0 < max_steps < global_step:
                    epoch_iterator.close()
                    break

            epoch_num += 1

            if 0 < max_steps < global_step:
                train_iterator.close()
                break

        return global_step, tr_loss / global_step

    def evaluate(self, task_name):
        # Loop to handle MNLI double evaluation (matched, mis-matched)
        eval_task_names = ("mnli", "mnli-mm") if task_name == "mnli" else (task_name,)
        eval_outputs_dirs = (self.output_dir, self.output_dir + '-MM') if task_name == 'mnli' else (self.output_dir,)

        results = {}
        for eval_task, eval_output_dir in zip(eval_task_names, eval_outputs_dirs):
            eval_dataset = self.load_and_cache_examples(eval_task, evaluate=True)
            if not os.path.exists(eval_output_dir):
                os.makedirs(eval_output_dir)
            eval_sampler = SequentialSampler(eval_dataset)
            eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=self.eval_batch_size)

            if self.n_gpu > 1:
                self.model = torch.nn.DataParallel(self.model)

            eval_loss = 0.0
            nb_eval_steps = 0
            preds = None
            out_label_ids = None

            for batch in tqdm(eval_dataloader, desc='Evaluating'):
                self.model.eval()
                batch = tuple(t.to(self.device) for t in batch)

                with torch.no_grad():
                    input_ids = batch[0]
                    attention_mask = batch[1]
                    token_type_ids = batch[2]
                    labels = batch[3]

                    outputs = model(input_ids)
                    outputs = torch.argmax(outputs, dim=-1).float()
                    tmp_eval_loss = self.loss_fn(outputs, labels.long())
                    eval_loss += tmp_eval_loss.mean().item()

                nb_eval_steps += 1
                if preds is None:
                    preds = torch.argmax(outputs, dim=-1).detach().cpu().numpy()
                    out_label_ids = labels.detach().cpu().numpy()
                else:
                    preds = np.append(preds, torch.argmax(outputs, dim=-1).detach().cpu().numpy())
                    out_label_ids = np.append(out_label_ids, labels.detach().cpu().numpy())

            eval_loss = eval_loss / nb_eval_steps
            if self.output_mode == 'classification':
                #preds = np.argmax(preds, axis=1)
                pass
            elif self.output_mode == 'regression':
                preds = np.squeeze(preds)
            result = compute_metrics(eval_task, preds, out_label_ids)
            results.update(result)

            output_eval_file = os.path.join(eval_output_dir, 'eval_results.txt')
            with open(output_eval_file, 'w') as writer:
                for key in sorted(result.keys()):
                    writer.write(f'{key} = {result[key]}')
        return results

    def load_and_cache_examples(self, task, evaluate=False):

        processor = processors[task]()
        output_mode = output_modes[task]
        cached_features_file = os.path.join(
            self.data_dir,
            "cached_{}_{}_{}".format(
                "dev" if evaluate else "train",
                # list(filter(None, self.model_name_or_path.split("/"))).pop(),
                str(self.max_seq_len),
                str(task),
            )
        )
        if os.path.exists(cached_features_file):
            features = torch.load(cached_features_file)
        else:
            label_list = processor.get_labels()
            examples = (
                processor.get_dev_examples(self.data_dir) if evaluate else processor.get_train_examples(self.data_dir)
            )

            features = convert_examples_to_features(
                examples,
                tokenizer,
                label_list=label_list,
                max_length=self.max_seq_len,
                output_mode=self.output_mode,
                pad_on_left=False,
                pad_token=self.tokenizer.convert_tokens_to_ids([self.tokenizer.pad_token])[0],
                pad_token_segment_id=0,
            )
            torch.save(features, cached_features_file)

        # converting to tensors and building datasets
        all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
        all_attention_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long)
        all_token_type_ids = torch.tensor([f.token_type_ids for f in features], dtype=torch.long)

        if self.output_mode == 'classification':
            all_labels = torch.tensor([f.label for f in features], dtype=torch.long)
        elif self.output_mode == 'regression':
            all_labels = torch.tensor([f.label for f in features], dtype=torch.float)
        else:
            all_labels = None

        dataset = TensorDataset(
            all_input_ids,
            all_attention_mask,
            all_token_type_ids,
            all_labels
        )
        return dataset

## -- main --

parser = argparse.ArgumentParser(description='Main GLUE trainer')
parser.add_argument('--tasks', default = None)

parser.add_argument('--gpu', action='store_true')

parser.add_argument('--max_seq_len', type=int, default = 512)
parser.add_argument('--n_hashes', type=int)
parser.add_argument('--causal', action='store_true')
parser.add_argument('--tied_connections', action='store_true')
parser.add_argument('--kmeans', action='store_true')
parser.add_argument('--full_attention', action='store_true')

parser.add_argument('--ckpt_dir', type=str, default = "")

parser.add_argument('--num_train_epochs', type=int, default = 10)
parser.add_argument('--num_eval_steps', type=int, default = 20)
args = parser.parse_args()

device = torch.device("cuda" if torch.cuda.is_available() and args.gpu else "cpu")

tokenizer = BertTokenizer.from_pretrained('bert-base-cased')
tokenizer.max_len = args.max_seq_len

model = ReformerLM(
    num_tokens      = tokenizer.vocab_size,
    dim             = 512,
    depth           = 6,
    heads           = 1,
    n_hashes        = args.n_hashes,
    max_seq_len     = args.max_seq_len,
    causal          = args.causal,
    recurrence      = args.tied_connections,
    k_means_hashing = args.kmeans,
    use_full_attn   = args.full_attention
).to(device)

if args.ckpt_dir is not "":
    assert os.path.isdir(args.ckpt_dir)
    try:
        logging.info(f'{datetime.now()} | Continuing from checkpoint...')
        model.load_state_dict(torch.load(f'{ckpt_dir}/model_state_dict.pt', map_location=device))
        print('Model loaded!')
    except Exception as e:
        logging.info(f'{datetime.now()} | No checkpoint was found | {e}')

# training on glue tasks
for key in list(args.tasks.split(',')) if args.tasks is not None else processors.keys():
    task_name = key.lower()
    processor = processors[task_name]()
    output_mode = output_modes[task_name]
    label_list = processor.get_labels()
    num_labels = len(label_list)

    print("Training task " + task_name + " with output mode " + output_mode)

    trainer = TrainerGLUE(
        model=model,
        max_seq_len=args.max_seq_len,
        tokenizer=tokenizer,
        task=key,
        per_gpu_train_batch_size=32,
        per_gpu_eval_batch_size=8,
        num_tokens = tokenizer.vocab_size,
        output_dim = num_labels
    )

    train_dataset = trainer.load_and_cache_examples(task_name, tokenizer)
    print('Dataset loaded')
    trainer.train(train_dataset, task_name=task_name, logging_steps=args.num_eval_steps, num_train_epochs=args.num_train_epochs)
    print(f'Trained: {task_name}')


import numpy as np
import cv2 as cv
import os

def detect_faces(image, f_scale=1.3, f_near=2):
    faces = []
    face_cascade = cv.CascadeClassifier('model/haarcascade_frontalface_default.xml')
    if os.path.isdir('temp'):
        pass
    else:
        os.makedirs('temp')
    img = cv.imread(image)
    gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)

    face_detector = face_cascade.detectMultiScale(gray, f_scale, f_near)
    n = 0
    for (x,y,w,h) in face_detector:
        n = n + 1
        cv.rectangle(img,(x,y),(x+w,y+h),(255,0,0),2)
        #roi_gray = gray[y:y+h, x:x+w]
        #roi_color = img[y:y+h, x:x+w]
        crop_face = img[y:y+h, x:x+w]
        faces.append(crop_face)
        cv.imwrite('temp/{}.jpg'.format(n),crop_face)
    print('{} faces detected'.format(len(faces)))


from bs4 import BeautifulSoup
import requests

# 告诉网站我们是一个合法的浏览器
headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.8",
    "Connection": "close",
    "Cookie": "_gauges_unique_hour=1; _gauges_unique_day=1; _gauges_unique_month=1; _gauges_unique_year=1; _gauges_unique=1",
    "Referer": "http://www.infoq.com",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36 LBBROWSER"
}

url = 'http://www.infoq.com/cn/news'

# 获取新闻标题
def craw2(url):

    print(current_thread().getName(), 'start')
    response = requests.get(url,headers = headers)
    soup = BeautifulSoup(response.text,'lxml')

    for title_href in soup.find_all('div',class_='news_type_block'):
        print([title.get('title')
               for title in title_href.find_all('a') if title.get('title')])
    print("-------------------------------------")

import threading
from threading import current_thread

# 翻页
for i in range(15,80,15):

    threadCraw = threading.Thread(target=craw2, args=(url,))
    threadCraw.start()
    url = 'http://www.infoq.com/cn/news' + str(i)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
from django.conf import settings
import uuid

class Migration(migrations.Migration):

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]

    operations = [
        migrations.CreateModel(
            name='Assignment',
            fields=[
                ('id', models.UUIDField(editable=False, primary_key=True, default=uuid.uuid4, serialize=False)),
                ('title', models.TextField()),
                ('total_value', models.DecimalField(decimal_places=5, max_digits=10)),
                ('is_discrete', models.BooleanField()),
                ('upper_bound', models.DecimalField(default=0.0, max_digits=10, decimal_places=5)),
                ('lower_bound', models.DecimalField(default=100.0, max_digits=10, decimal_places=5)),
            ],
            options={
                'verbose_name_plural': 'assignments',
                'verbose_name': 'assignment',
            },
        ),
        migrations.CreateModel(
            name='AssignmentType',
            fields=[
                ('id', models.UUIDField(editable=False, primary_key=True, default=uuid.uuid4, serialize=False)),
                ('name', models.TextField()),
                ('weight', models.DecimalField(decimal_places=5, max_digits=10)),
            ],
            options={
                'verbose_name_plural': 'assignments types',
                'verbose_name': 'assignment type',
            },
        ),
        migrations.CreateModel(
            name='ContinuousScore',
            fields=[
                ('id', models.UUIDField(editable=False, primary_key=True, default=uuid.uuid4, serialize=False)),
                ('value', models.DecimalField(decimal_places=5, max_digits=10)),
                ('assignment', models.ForeignKey(to='score_book.Assignment')),
            ],
            options={
                'verbose_name_plural': 'scores',
                'verbose_name': 'score',
            },
        ),
        migrations.CreateModel(
            name='Course',
            fields=[
                ('id', models.UUIDField(editable=False, primary_key=True, default=uuid.uuid4, serialize=False)),
                ('year', models.IntegerField()),
                ('full_name', models.TextField()),
                ('abbr_name', models.TextField()),
                ('location', models.TextField()),
            ],
            options={
                'verbose_name_plural': 'courses',
                'verbose_name': 'course',
            },
        ),
        migrations.CreateModel(
            name='DiscreteScore',
            fields=[
                ('id', models.UUIDField(editable=False, primary_key=True, default=uuid.uuid4, serialize=False)),
                ('assignment', models.ForeignKey(to='score_book.Assignment')),
            ],
            options={
                'verbose_name_plural': 'scores',
                'verbose_name': 'score',
            },
        ),
        migrations.CreateModel(
            name='DiscreteScoreSystem',
            fields=[
                ('id', models.UUIDField(editable=False, primary_key=True, default=uuid.uuid4, serialize=False)),
                ('course', models.ForeignKey(to='score_book.Course')),
            ],
            options={
                'verbose_name_plural': 'score systems',
                'verbose_name': 'score system',
            },
        ),
        migrations.CreateModel(
            name='DiscreteScoreType',
            fields=[
                ('id', models.UUIDField(editable=False, primary_key=True, default=uuid.uuid4, serialize=False)),
                ('numeric_value', models.DecimalField(decimal_places=5, max_digits=10)),
                ('text_value', models.TextField()),
                ('image_value', models.ImageField(upload_to='')),
                ('real_value', models.DecimalField(decimal_places=5, max_digits=10)),
                ('course', models.ForeignKey(to='score_book.Course')),
                ('score_system', models.ManyToManyField(to='score_book.DiscreteScoreSystem')),
            ],
            options={
                'verbose_name_plural': 'score types',
                'verbose_name': 'score type',
            },
        ),
        migrations.CreateModel(
            name='Instructor',
            fields=[
                ('id', models.UUIDField(editable=False, primary_key=True, default=uuid.uuid4, serialize=False)),
                ('ucinetid', models.TextField()),
                ('preferred_name', models.TextField(default='')),
                ('courses', models.ManyToManyField(to='score_book.Course')),
            ],
            options={
                'ordering': ['ucinetid'],
                'verbose_name': 'instructor',
                'verbose_name_plural': 'instructors',
            },
        ),
        migrations.CreateModel(
            name='Quarter',
            fields=[
                ('id', models.UUIDField(editable=False, primary_key=True, default=uuid.uuid4, serialize=False)),
                ('name', models.TextField()),
                ('number', models.IntegerField()),
            ],
            options={
                'ordering': ['number'],
                'verbose_name': 'quarter',
                'verbose_name_plural': 'quarters',
            },
        ),
        migrations.CreateModel(
            name='Section',
            fields=[
                ('id', models.UUIDField(editable=False, primary_key=True, default=uuid.uuid4, serialize=False)),
                ('name', models.TextField()),
                ('number', models.IntegerField()),
                ('course', models.ForeignKey(to='score_book.Course')),
                ('instructors', models.ManyToManyField(to='score_book.Instructor')),
            ],
            options={
                'verbose_name_plural': 'sections',
                'verbose_name': 'section',
            },
        ),
        migrations.CreateModel(
            name='Student',
            fields=[
                ('id', models.UUIDField(editable=False, primary_key=True, default=uuid.uuid4, serialize=False)),
                ('ucinetid', models.TextField()),
                ('preferred_name', models.TextField(default='')),
                ('studentid', models.PositiveIntegerField()),
                ('courses', models.ManyToManyField(to='score_book.Course')),
                ('sections', models.ManyToManyField(to='score_book.Section')),
                ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
            ],
            options={
                'ordering': ['ucinetid'],
                'verbose_name': 'student',
                'verbose_name_plural': 'students',
            },
        ),
        migrations.AddField(
            model_name='section',
            name='students',
            field=models.ManyToManyField(to='score_book.Student'),
        ),
        migrations.AddField(
            model_name='instructor',
            name='sections',
            field=models.ManyToManyField(to='score_book.Section'),
        ),
        migrations.AddField(
            model_name='instructor',
            name='user',
            field=models.ForeignKey(to=settings.AUTH_USER_MODEL),
        ),
        migrations.AddField(
            model_name='discretescoresystem',
            name='possible_values',
            field=models.ManyToManyField(to='score_book.DiscreteScoreType'),
        ),
        migrations.AddField(
            model_name='discretescore',
            name='student',
            field=models.ForeignKey(to='score_book.Student'),
        ),
        migrations.AddField(
            model_name='discretescore',
            name='value',
            field=models.ForeignKey(to='score_book.DiscreteScoreType'),
        ),
        migrations.AddField(
            model_name='course',
            name='instructors',
            field=models.ManyToManyField(to='score_book.Instructor'),
        ),
        migrations.AddField(
            model_name='course',
            name='quarter',
            field=models.ForeignKey(to='score_book.Quarter'),
        ),
        migrations.AddField(
            model_name='course',
            name='students',
            field=models.ManyToManyField(to='score_book.Student'),
        ),
        migrations.AddField(
            model_name='continuousscore',
            name='student',
            field=models.ForeignKey(to='score_book.Student'),
        ),
        migrations.AddField(
            model_name='assignmenttype',
            name='course',
            field=models.ForeignKey(to='score_book.Course'),
        ),
        migrations.AddField(
            model_name='assignment',
            name='assignment_type',
            field=models.ForeignKey(to='score_book.AssignmentType'),
        ),
        migrations.AddField(
            model_name='assignment',
            name='course',
            field=models.ForeignKey(to='score_book.Course'),
        ),
        migrations.AddField(
            model_name='assignment',
            name='discrete_score_system',
            field=models.ForeignKey(to='score_book.DiscreteScoreSystem'),
        ),
    ]


# -*- coding: utf-8 -*-
"""
Dubalu Framework
~~~~~~~~~~~~~~~~

URLconf for registration and activation, using django-registration's
default backend.

If the default behavior of these views is acceptable to you, simply
use a line like this in your root URLconf to set up the default URLs
for registration::

    (r'^accounts/', include('dfw.registration.backends.default.urls')),

This will also automatically set up the views in
``django.contrib.auth`` at sensible default locations.

If you'd like to customize the behavior (e.g., by passing extra
arguments to the various views) or split up the URLs, feel free to set
up your own URL patterns for these views instead.

:author: Dubalu Framework Team. See AUTHORS.
:copyright: Copyright (c) 2013-2014, deipi.com LLC. All Rights Reserved.
:copyright: Copyright (c) 2007-2012, James Bennett.
:license: See LICENSE for license details.

"""
from __future__ import absolute_import, unicode_literals

from django.conf.urls import patterns, include, url

from ...views import activate, register, direct_to_template

def get_urlpatterns(backend='dfw.registration.backends.default.DefaultBackend'):
    return patterns('',
        url(
            r'^activate/complete/$',
            direct_to_template,
            {
                'template_name': 'registration/activate.html',
                'backend': backend,
                'extra_context': {'account': True},
            },
            name='registration_activation_complete',
        ),
        # Activation keys get matched by \w+ instead of the more specific
        # [a-fA-F0-9]{40} because a bad activation key should still get to the view;
        # that way it can return a sensible "invalid key" message instead of a
        # confusing 404.
        url(
            r'^activate/(?P<activation_key>\w+)/$',
            activate,
            {
                'backend': backend,
            },
            name='registration_activate',
        ),
        url(
            r'^register/$',
            register,
            {
                'backend': backend,
            },
            name='registration_register',
        ),
        url(
            r'^register/complete/$',
            direct_to_template,
            {
                'template_name': 'registration/registration_complete.html',
                'backend': backend,
            },
            name='registration_complete'),
        url(
            r'^register/closed/$',
            direct_to_template,
            {
                'template_name': 'registration/registration_closed.html',
                'backend': backend,
            },
            name='registration_disallowed',
        ),
        (r'', include('dfw.registration.auth_urls')),
    )

urlpatterns = get_urlpatterns()


"""Can we make windrose plots?"""

import pytest
import numpy as np
from metpy.units import units
from pyiem.plot.windrose import WindrosePlot, histogram, plot

BINS = np.array([2, 5, 10, 20]) * units("mph")
SPEED = np.arange(0, 60, 0.1) * units("meter / second")
DIRECTION = SPEED.m * 6.0 * units("degree")

@pytest.mark.mpl_image_compare(tolerance=0.1)
def test_windrose_basic():
    """Can we crawl."""
    wr = WindrosePlot()
    return wr.fig

@pytest.mark.mpl_image_compare(tolerance=0.1)
def test_windrose_barplot_noopts():
    """Generate a plot with defaults for bins."""
    wr = plot(DIRECTION, SPEED, rmax=10)
    return wr.fig

@pytest.mark.mpl_image_compare(tolerance=0.1)
def test_windrose_barplot():
    """Can we crawl."""
    wr = plot(DIRECTION, SPEED, bins=BINS, nsector=16)
    return wr.fig

def test_histogram():
    """Can we make histograms properly."""
    nsector = 16
    calm_percent, dirbins, table = histogram(SPEED, DIRECTION, BINS, nsector)
    # 9 out of 600 above are below 2 MPH, 1.5%
    assert abs(calm_percent.m - 1.50) < 0.01
    assert dirbins.m.shape[0] == table.m.shape[0]


#!/usr/bin/env python
# encoding: utf-8
"""
variant_annotator.py

Consumes batches of variants and annotates them. Each batch is a dictionary
with variant_id:s as keys and dictionaries with variant information.
The variants will get different annotations depending on input

Created by Måns Magnusson on 2013-03-01.
Copyright (c) 2013 __MyCompanyName__. All rights reserved.
"""

from __future__ import (division, print_function, absolute_import)

import sys
import os
import operator
import tabix
import logging

from multiprocessing import Process
from math import log10

from genmod.vcf_tools import get_variant_dict, add_vcf_info
from genmod.annotate_regions import (get_genes, check_exonic)
from genmod.annotate_variants import (get_frequencies, get_cadd_scores,
get_spidex_score, get_cosmic)

# from . import (annotate_cadd_score, annotate_frequency, check_genetic_models)

class VariantAnnotator(Process):
    """
    Annotates variant in batches from the task queue and puts the result in
    the results queue.
    """

    def __init__(self, task_queue, results_queue, **kwargs):

        Process.__init__(self)
        self.logger = logging.getLogger(__name__)

        self.proc_name = self.name

        self.logger.info("Setting up variant_annotator: {0}".format(
            self.proc_name))
        self.logger.debug("Setting up task queue")

        self.task_queue = task_queue

        self.logger.debug("Setting up results queue")
        self.results_queue = results_queue

        self.header_line = kwargs.get('header_line')
        if not self.header_line:
            raise IOError("VariantAnnotator needs a vcf header line")

        self.cadd_raw = kwargs.get('cadd_raw', None)
        self.logger.debug("Setting cadd raw to {0}".format(self.cadd_raw))

        self.max_af = kwargs.get('max_af', None)
        self.logger.debug("Setting max_af to {0}".format(self.max_af))

        self.spidex = kwargs.get('spidex', None)
        self.logger.debug("Setting spidex to {0}".format(self.spidex))

        self.cosmic = kwargs.get('cosmic', None)
        self.logger.debug("Setting cosmic to {0}".format(self.spidex))

        self.exon_trees = kwargs.get('exon_trees', {})
        self.gene_trees = kwargs.get('gene_trees', {})

        ######### Annotation files #########
        # Cadd files #
        self.cadd_handles = []
        for cadd_path in kwargs.get('cadd_files', []):
            self.logger.debug("Opening cadd file {0} with tabix open".format(
                cadd_path
            ))
            self.cadd_handles.append(tabix.open(cadd_path))
            self.logger.debug("Cadd file opened")

        # Frequency files #
        self.thousand_g = kwargs.get('thousand_g', None)
        self.logger.debug("1000G frequency file {0}".format(self.thousand_g))
        self.exac = kwargs.get('exac', None)
        self.logger.debug("Exac frequency file {0}".format(self.thousand_g))
        # self.dbNSFP = kwargs.get('dbNSFP', None)

        if self.thousand_g:
            self.logger.debug("Opening 1000G frequency file with tabix open")
            self.thousand_g = tabix.open(self.thousand_g)
            self.logger.debug("1000G frequency file opened")
        if self.exac:
            self.logger.debug("Opening ExAC frequency file with tabix open")
            self.exac = tabix.open(self.exac)
            self.logger.debug("ExAC frequency file opened")
        if self.spidex:
            self.logger.debug("Opening Spidex database with tabix open")
            self.spidex = tabix.open(self.spidex)
            self.logger.debug("Spidex file opened")
        if self.cosmic:
            self.logger.debug("Opening COSMIC database with tabix open")
            self.cosmic = tabix.open(self.cosmic)
            self.logger.debug("COSMIC file opened")

    def run(self):
        """Run the consuming"""
        self.logger.info('%s: Starting!' % self.proc_name)
        # Check if there are any batches in the queue
        while True:
            # A batch is a dictionary on the form {variant_id:variant_dict}
            variant_line = self.task_queue.get()

            if variant_line is None:
                self.logger.info('No more variants')
                self.task_queue.task_done()
                self.logger.info('{0}: Exiting'.format(self.proc_name))
                break

            variant_dict = get_variant_dict(
                variant_line = variant_line,
                header_line = self.header_line
            )

            chrom = variant_dict['CHROM'].lstrip('chr')
            position = int(variant_dict['POS'])
            ref = variant_dict['REF']
            alternative = variant_dict['ALT']

            longest_alt = max([
                len(alt) for alt in alternative.split(',')])

            if self.exon_trees:
                exonic = check_exonic(
                    chrom = chrom,
                    start = position,
                    stop = (position+longest_alt)-1,
                    exon_trees = self.exon_trees
                )
                if exonic:
                    variant_dict = add_vcf_info(
                            keyword = 'Exonic',
                            variant_dict=variant_dict,
                            annotation=None
                            )

            if self.gene_trees:
                genes = get_genes(
                    chrom = chrom,
                    start = position,
                    stop = (position+longest_alt)-1,
                    gene_trees = self.gene_trees
                )
                if genes:
                    variant_dict = add_vcf_info(
                        keyword = "Annotation",
                        variant_dict = variant_dict,
                        annotation = ','.join(genes)
                    )
            if self.thousand_g:

                frequencies = get_frequencies(
                    tabix_reader = self.thousand_g,
                    chrom=chrom,
                    start=position,
                    alt=alternative
                )
                if frequencies.get('AF'):
                    variant_dict = add_vcf_info(
                        keyword = "1000GAF",
                        variant_dict = variant_dict,
                        annotation = frequencies['AF']
                    )
                if self.max_af:
                    if frequencies.get('MAX_AF'):
                        variant_dict = add_vcf_info(
                            keyword = "1000G_MAX_AF",
                            variant_dict = variant_dict,
                            annotation = frequencies['MAX_AF']
                            )

            if self.exac:

                frequencies = get_frequencies(
                    tabix_reader = self.exac,
                    chrom=chrom,
                    start=position,
                    alt=alternative
                )
                if frequencies.get('AF'):
                    variant_dict = add_vcf_info(
                        keyword = "ExACAF",
                        variant_dict = variant_dict,
                        annotation = frequencies['AF']
                    )
                if self.max_af:
                    if frequencies.get('MAX_AF'):
                        variant_dict = add_vcf_info(
                            keyword = "ExAC_MAX_AF",
                            variant_dict = variant_dict,
                            annotation = frequencies['MAX_AF']
                        )

            if self.spidex:
                spidex_score = get_spidex_score(
                        tabix_reader=self.spidex,
                        chrom=chrom,
                        start=position,
                        alt=alternative
                    )
                if spidex_score:
                    variant_dict = add_vcf_info(
                        keyword = "SPIDEX",
                        variant_dict = variant_dict,
                        annotation = spidex_score
                    )

            if self.cosmic:
                in_cosmic = get_cosmic(
                        tabix_reader=self.cosmic,
                        chrom=chrom,
                        start=position,
                        alt=alternative
                    )
                if in_cosmic:
                    variant_dict = add_vcf_info(
                        keyword = "COSMIC",
                        variant_dict = variant_dict,
                    )

            if self.cadd_handles:

                cadd_phred = None
                cadd_raw = None
                for cadd_handle in self.cadd_handles:

                    cadd_scores = get_cadd_scores(
                        tabix_reader=cadd_handle,
                        chrom=chrom,
                        start=position,
                        alt=alternative
                    )
                    if cadd_scores['cadd_phred']:
                        cadd_phred = cadd_scores['cadd_phred']
                        cadd_raw = cadd_scores['cadd_raw']

                        variant_dict = add_vcf_info(
                            keyword = "CADD",
                            variant_dict = variant_dict,
                            annotation = cadd_phred
                        )

                        if self.cadd_raw and cadd_raw:
                            variant_dict = add_vcf_info(
                                keyword = "CADD_raw",
                                variant_dict = variant_dict,
                                annotation = cadd_raw
                            )
                        break

            self.results_queue.put(variant_dict)
            self.task_queue.task_done()

        return

def main():
    pass

if __name__ == '__main__':
    main()


from flask import Blueprint, abort, jsonify, redirect, request

from api.downloaders import SpotifyDownloader
from core import config, const, spotdl
from core.const import log

app = Blueprint('/', 'main')

@app.errorhandler(400)
def custom400(error):
    res = jsonify({'error': error.description})
    res.status_code = 400
    return res

@app.errorhandler(500)
def custom500(error):
    res = jsonify({'error': error.description})
    res.status_code = 500
    return res

@app.route('/', methods=['GET'])
def index():
    return redirect("/index.html", code=302)

current_downloads = {}

@app.route('/download', methods=['POST'])
def post_download():
    if not request.json or 'url' not in request.json:
        abort(400, 'Error downloading playlist info: url obligatory')

    try:
        url = request.json['url']

        log.info("Downloading url[%s]", url)

        if url not in current_downloads:
            downloader = SpotifyDownloader(url)

            downloader.start()

            current_downloads[url] = downloader

            return jsonify({'status': 'OK'})
        else:
            return jsonify({'status': 'ALREADY_ADDED'})
    except Exception:
        log.error("Error downloading playlist info", exc_info=True)

        abort(400, 'Error downloading playlist info')

@app.route('/info', methods=['POST'])
def info():
    if not request.json or 'url' not in request.json:
        abort(400, 'Error getting playlist info: url obligatory')

    try:
        url = request.json['url']

        log.info("Get playlist info[%s]", url)

        info = spotdl.fetch_info(url)

        return jsonify(info)
    except Exception:
        log.error("Error getting playlist info", exc_info=True)

        abort(500, 'Error getting playlist info')

@app.route('/youtube', methods=['GET'])
def youtube():
    if 'url' not in request.args:
        abort(400,
              'Error getting playlist redirecting to youtube: url obligatory')

    try:
        url = request.args['url']

        log.info("Redirecting to youtube[%s]", url)

        yt_url = spotdl.fetch_yt_url(url)

        return redirect(yt_url, code=302)
    except Exception:
        log.error("Error redirecting to youtube", exc_info=True)

        abort(500, 'Error redirecting to youtube')

@app.route('/search', methods=['POST'])
def search():
    if not request.json or 'query' not in request.json:
        abort(400, 'Error searching: query obligatory')

    try:
        query = request.json['query']

        log.info("Searching[%s]", query)

        items = spotdl.search(query, max_results_per_type=int(
            const.config.search_max_results))

        return jsonify(items)
    except Exception:
        log.error("Error searching", exc_info=True)

        abort(500, 'Error searching')

@app.route('/download_history', methods=['GET'])
def download_history():
    log.info("Retrieving download_history")

    try:
        items = list(map(lambda d: {
            'url': d[0],
            'name': d[1].get_name(),
            'status': d[1].get_status().__dict__,
            'init_date': d[1].get_init_date(format="%H:%M:%S"),
            'end_date': d[1].get_end_date(format="%H:%M:%S"),
        },
            current_downloads.items()))

        response = {
            'items': items
        }

        return jsonify(response)
    except Exception:
        log.error("Error retrieving download_history", exc_info=True)

        abort(500, 'Error retrieving download_history')

@app.route('/settings', methods=['GET'])
def get_settings():
    log.info("Retrieving settings")

    try:
        settings = {
            'settings': config.get_config_dict()
        }

        return jsonify(settings)
    except Exception:
        log.error("Error Retrieving settings", exc_info=True)

        abort(500, 'Error Retrieving settings')

@app.route('/settings', methods=['PUT'])
def put_settings():
    if not request.json or 'settings' not in request.json:
        abort(400, 'Error Saving settings: settings obligatory')

    log.info("Saving settings")

    try:
        config.save_config(request.json['settings'])

        config.init_config()

        return jsonify({'status': 'OK'})
    except Exception:
        log.error("Error Saving settings", exc_info=True)

        abort(500, 'Error Saving settings')


"""URL's for the chat app."""

from django.contrib import admin
from django.urls import path

from . import views

urlpatterns = [
    path('chats/', views.ChatSessionView.as_view()),
    path('chats/<uri>/', views.ChatSessionView.as_view()),
    path('my-chats/', views.MyChatSessionsView.as_view()),
    path('received/', views.MessageReceived.as_view()),
    path('seen/', views.MessageSeen.as_view()),
    path('test-chat/', views.ChatBot.as_view()),
    path('chats/<uri>/messages/', views.ChatSessionMessageView.as_view()),
    path('start-video-chat/', views.VideoChat.as_view()),
]