from torch.utils.data import Dataset, DataLoader
import json
import numpy as np
import torch

CATEGORIES = [
    'artifact', 'transferownership', 'transaction', 'broadcast', 'contact', 'demonstrate', \
    'injure', 'transfermoney', 'transportartifact', 'attack', 'meet', 'elect', \
    'endposition', 'correspondence', 'arrestjail', 'startposition', 'transportperson', 'die'
]

class KBPTrigger(Dataset):
    def __init__(self, data_file):
        self.data = self.load_data(data_file)

    def load_data(self, data_file):
        Data = []
        with open(data_file, 'rt', encoding='utf-8') as f:
            for line in f:
                sample = json.loads(line.strip())
                tags = [
                    (event['start'], event['start'] + len(event['trigger']) - 1, event['trigger'], event['subtype'])
                    for event in sample['events'] if event['subtype'] in CATEGORIES
                ]
                Data.append({
                    'id': sample['doc_id'],
                    'document': sample['document'],
                    'tags': tags
                })
        return Data

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        return self.data[idx]

def get_dataLoader(args, dataset, tokenizer, batch_size=None, shuffle=False):

    def collote_fn(batch_samples):
        batch_sentence, batch_tags  = [], []
        for sample in batch_samples:
            batch_sentence.append(sample['document'])
            batch_tags.append(sample['tags'])
        batch_inputs = tokenizer(
            batch_sentence,
            max_length=args.max_seq_length,
            padding=True,
            truncation=True,
            return_tensors="pt"
        )
        batch_label = np.zeros(batch_inputs['input_ids'].shape, dtype=int)
        for s_idx, sentence in enumerate(batch_sentence):
            encoding = tokenizer(sentence, max_length=args.max_seq_length, truncation=True)
            for char_start, char_end, _, tag in batch_tags[s_idx]:
                token_start = encoding.char_to_token(char_start)
                token_end = encoding.char_to_token(char_end)
                if not token_start or not token_end:
                    continue
                batch_label[s_idx][token_start] = args.label2id[f"B-{tag}"]
                batch_label[s_idx][token_start + 1:token_end + 1] = args.label2id[f"I-{tag}"]
        batch_inputs['labels'] = torch.tensor(batch_label)
        return batch_inputs

    return DataLoader(dataset, batch_size=(batch_size if batch_size else args.batch_size), shuffle=shuffle,
                      collate_fn=collote_fn)


#! /usr/bin/env python3

import argparse
import string
import sys

import nltk

if __name__ == '__main__':
    # Argument parser
    parser = argparse.ArgumentParser(
        prog='count_words',
        description='Count words in a text document'
    )
    parser.add_argument('filename',
        help='text file that contains the text to be analyzed'
    )
    parser.add_argument('--sort',
        choices=['alpha', 'freq'],
        default='freq',
        help='(default: %(default)s) '
             'sort order in which the results are to be returned'
    )
    parser.add_argument('--stemmer',
        choices=['on', 'off'],
        default='on',
        help='(default: %(default)s) '
             'turn the Porter stemmer on or off'
    )
    parser.add_argument('--syn-file',
        help='path to the synonyms file. '
             'If it is not given, no synonym replacement will be done'
    )
    parser.add_argument('--stopwords-file',
        help='path to the stopwords file. '
             'If it is not given, no stopwords will be removed'
    )
    args = parser.parse_args()

    # Read document
    with open(args.filename, 'r') as infile:
        document = infile.readlines()

    # Basic preprocessing
    for i in range(len(document)):
        new_line = document[i].translate(str.maketrans('', '', string.punctuation))
        new_line = new_line.strip().lower()
        document[i] = new_line

    # Synonym replacement
    if args.syn_file is not None:
        try:
            # Read synonyms file
            synonyms = {}
            with open(args.syn_file, 'r') as infile:
                for line in infile:
                    target, _,  terms = line.partition(':')
                    target = target.strip()
                    terms = terms.split(',')
                    for term in terms:
                        synonyms[term.strip()] = target

            # Replace synonyms
            for i in range(len(document)):
                new_line = document[i]
                for term, repl in synonyms.items():
                    new_line = new_line.replace(term, repl)
                document[i] = new_line
        except FileNotFoundError:
            print(f"ERROR: Synonyms file '{args.syn_file}' not found.", file=sys.stderr)
            exit()

    # Read stopwords file
    if args.stopwords_file is not None:
        try:
            with open(args.stopwords_file, 'r') as infile:
                stopwords = {line.strip() for line in infile if line}
        except FileNotFoundError:
            print(f"ERROR: Stopwords file '{args.stopwords_file}' not found.", file=sys.stderr)
            exit()
    else:
        stopwords = set()

    # Initialize Porter stemmer
    if args.stemmer == 'on':
        stemmer = nltk.stem.porter.PorterStemmer()

    # Count words
    word_counts = {}
    for line in document:
        for word in line.split():
            if args.stemmer == 'on':
                stem = stemmer.stem(word)
            else:
                stem = word
            if stem in stopwords:
                continue
            if stem not in word_counts:
                word_counts[stem] = 0
            word_counts[stem] += 1

    # Output result
    if args.sort == 'freq':
        sort_key = lambda x: (-x[1], x[0])
    else:
        sort_key = lambda x: (x[0], -x[1])
    for word, cnt in sorted(word_counts.items(), key=sort_key):
        print(f'{cnt:4d} - {word}')


from app import tracer, _logger

from flask import Blueprint, jsonify, request, Response
from opentelemetry.trace import set_span_in_context
from opentelemetry.trace.status import StatusCode
import json
import traceback
import requests

apis_bp = Blueprint('apis_bp', __name__)

@apis_bp.route("/api", methods=["GET"])
def my_api():
    answer = "something"
    return jsonify(repsonse=answer)

@apis_bp.route("/api/entries", methods=["GET"])
def get_public_entries():
    if request.args.get('type', None) == "public":
        homepages = []
        succeeded = False
        entries = []
        with tracer.start_span('get-public-apis') as span:
            try:
                SAMPLE_COUNT = int(request.args.get('sample', 30))
                TOTAL_COUNT = -1
                ENTRY_TIMEOUT = float(request.args.get('entry-timeout', 2.5))
                ctx = set_span_in_context(span)
                res = requests.get(
                    'https://api.publicapis.org/entries', timeout=3)
                if res.status_code == 200:
                    TOTAL_COUNT = res.json()['count']
                    _logger.info(
                        f"Received {TOTAL_COUNT} total entries")
                    _logger.info(
                        f"Collecting homepages for the first {SAMPLE_COUNT} entries to save time with timeout {ENTRY_TIMEOUT} per entry connection")
                    span.set_attribute('count.total', TOTAL_COUNT)
                    span.set_attribute('count.sample', SAMPLE_COUNT)
                    for entry in res.json()['entries'][:SAMPLE_COUNT]:
                        _logger.info(
                            f"Collecting homepage for api='{entry['API']}' category='{entry['Category']}' link='{entry['Link']}'")
                        with tracer.start_span(entry['API'], context=ctx) as api_span:
                            try:
                                homepages.append({
                                    'link': entry['Link'],
                                    'page': requests.get(entry['Link'], timeout=ENTRY_TIMEOUT)
                                })
                                entries.append(entry)
                                api_span.set_attribute('status', 'SUCCESS')
                                api_span.set_attribute(
                                    'category', entry['Category'])

                                api_span.set_attribute('cors', entry['Cors']) if entry['Cors'] else api_span.set_attribute(
                                    'cors', "N/A")
                                api_span.set_attribute('auth', entry['Auth']) if entry['Auth'] else api_span.set_attribute(
                                    'auth', "N/A")
                                api_span.set_attribute('https', entry['HTTPS'])
                            except Exception as e:
                                api_span.add_event(
                                    "exception", {"exception.message": str(e), "exception.stacktrace": traceback.format_exc()})
                                api_span.set_status(StatusCode.ERROR)
                                _logger.error(
                                    f"Error: unable to get homepage for '{entry['API']}' reason='{str(e)}'")
                                api_span.set_attribute('status', 'FAILED')
                    span.add_event("collection-success", {"homepages.count.collected": len(
                        homepages), "homepages.count.total": TOTAL_COUNT, "homepages.count.sample": SAMPLE_COUNT})
                    succeeded = True
                    _logger.info(f"Collected {len(homepages)} homepages")
                else:
                    SAMPLE_COUNT = -1
                    _logger.error(
                        f"Error: expected status code '200' got '{res.status_code}' instead")
                    _logger.info("Nothing to do..")
                    span.add_event("exception", {
                                   "exception.message": f"Error: expected status code '200' got '{res.status_code}' instead"})
                    span.set_status(StatusCode.ERROR)

            except Exception as e:
                SAMPLE_COUNT = -1
                _logger.error(f"Error: {str(e)}")
                span.add_event("exception", {"exception.message": str(
                    e), "exception.stacktrace": traceback.format_exc()})
                span.set_status(StatusCode.ERROR)

            return {
                "succeeded": succeeded,
                "results": {
                    "collected": len(homepages),
                    "sample": SAMPLE_COUNT,
                    "total": TOTAL_COUNT,
                    "entries": entries
                }
            }
    error = {
        'status_code': 400,
        'message': f"Error: Expected '?type=public' got '?type={request.args.get('type')}'"
    }
    _logger.error(error['message'])
    return Response(response=json.dumps(
        {'error': error['message']}), status=error['status_code'], mimetype='application/json')


import json
from decimal import Decimal

from django.conf import settings
from django.db.models import QuerySet

from applications.orders.models import Order, OrderItem
from applications.store.models import Product
from .services import messages

class Cart:

    def __init__(self, session):

        self.session = session
        if not self.session.get(settings.CART_SESSION_ID):
            self.session[settings.CART_SESSION_ID] = {}
        self.cart = self.session[settings.CART_SESSION_ID]
        self.sorted_cart = {}
        self.warning_messages = []
        self.error_messages = []

    def add(self, product: dict, quantity: int):
        """
        Add product to db
        :param product: format is
             {"slug":{
                    "Image": str,
                    "Product": str,
                    "Price": str,
                    "Discount price": str),

                    }
            }
        :type product: dict
        :param quantity:
        :type quantity: int
        """
        slug = [key for key in product][0]

        if slug not in self.cart:
            product[slug]['Quantity'] = quantity
            self.cart |= product
            self.__set_total_product_price(slug, product)
        else:
            self.cart[slug]['Quantity'] += quantity
            self.__set_total_product_price(slug, product)

        self.__save()

    def remove(self, slug: str):
        """
        Removing an item from the cart.
        """
        if slug in self.cart:
            del self.cart[slug]
            self.__save()

    def sort(self, slugs: list[str]):
        """
        Sorting cart according to product order in db
        """
        self.sorted_cart = self.__sort_help(self.cart, slugs)
        self.__save()

    def check_products_amount(self, products: QuerySet[Product]):
        for slug, product in zip(self.sorted_cart.copy(), products):
            if self.sorted_cart[slug]['Quantity'] > product.amount:

                if not product.amount:
                    self.__product_is_over(product, slug)
                else:
                    self.__quantity_is_less_then_possible_amount(product, slug)

        self.cart = self.__sort_help(self.sorted_cart, list(self.cart.keys()))
        self.__save()

    def create_order_items(self, order: Order, products: QuerySet[Product]):
        self.check_products_amount(products)

        if not self.warning_messages:
            sorted_cart = self.sorted_cart.copy()
            for slug, product in zip(sorted_cart, products):
                quantity = self.sorted_cart[slug]['Quantity']
                OrderItem.objects.create(
                    order=order, product=product, quantity=quantity,
                    price=sorted_cart[slug]['Price'],
                    discount_price=sorted_cart[slug]['Discount price'],
                    total_price=sorted_cart[slug]['Total price'])

            return True
        else:

            self.error_messages.append(messages.ERROR)
            return False

    def get_total_sum(self):
        """
        Calculate the cost of all items in the shopping cart.
        """
        return sum(Decimal(self.cart[i]['Total price']) for i in self.cart)

    def clear(self):
        """
        Delete cart from session
        """
        self.cart = {}
        self.__save()

    def get_th_for_table(self):
        for slug in self.cart:
            return self.cart[slug].keys()

    def __save(self):
        """
        Cart session update
        """
        self.session[settings.CART_SESSION_ID] = self.cart
        self.session.modified = True

    @staticmethod
    def __sort_help(dict_: dict, list_: list) -> dict:
        """
        Sorting a dict_ according to list_
        Example:
        >>> d={\
        'main_key2': {'key': 'value'},\
        'main_key1': {'key': 'value'}\
        }
        >>> l = ["main_key2", "main_key1"]
        >>> __sort_help(d, l)
        {'main_key2': {'key': 'value'}, 'main_key1': {'key': 'value'}}

        """
        return dict(sorted(dict_.items(),
                           key=lambda item: list_.index(item[0])))

    def __set_total_product_price(self, slug: str, product: dict,
                                  is_sorted: bool = False):

        cart = self.cart if not is_sorted else self.sorted_cart
        cart[slug]['Total price'] = str(cart[slug]['Quantity'] * Decimal(
            product[slug]['Discount price']))

    def __product_is_over(self, product: Product, slug: str):
        msg = messages.PRODUCT_IS_OVER.format(product=product)
        self.warning_messages.append(msg)
        del self.sorted_cart[slug]

    def __quantity_is_less_then_possible_amount(self, product: Product,
                                                slug: str):
        msg = messages.QUANTITY_IS_LESS_THEN_POSSIBLE_AMOUNT.format(
            quan=self.cart[slug]['Quantity'], product=product,
            amount=product.amount)
        self.warning_messages.append(msg)
        self.sorted_cart[slug]['Quantity'] = product.amount
        self.__set_total_product_price(
            slug, json.loads(product.as_cart_item()), True)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import tempfile
import base64

from fuel.datasets import IndexableDataset
from fuel.streams import DataStream

from dictlearn.data import (
    LanguageModellingData, ExtractiveQAData, RandomSpanScheme)
from dictlearn.vocab import Vocabulary
from dictlearn.util import vec2str

from tests.util import TEST_TEXT, TEST_SQUAD_BASE64_HDF5_DATA

def test_languge_modelling_data():
    temp_dir = tempfile.mkdtemp()
    train_path = os.path.join(temp_dir, "train.txt")
    with open(train_path, 'w') as dst:
        print(TEST_TEXT, file=dst)

    data = LanguageModellingData(temp_dir, 'standard')

    # test without batches
    stream = data.get_stream('train')
    it = stream.get_epoch_iterator()
    example = next(it)
    # skip one
    example = next(it)
    assert len(example) == 1
    assert len(example[0]) == 4
    assert example[0][0][:5].tolist() == map(ord, '<bos>')
    assert example[0][1][:5].tolist() == [ord('d'), ord('e'), ord('f'), 0, 0]
    assert example[0][2][:5].tolist() == [ord('d'), ord('e'), ord('f'), 0, 0]
    assert example[0][3][:5].tolist() == [ord('x'), ord('y'), ord('z'), 0, 0]

    # test with batches
    stream = data.get_stream('train', batch_size=2)
    it = stream.get_epoch_iterator()
    example = next(it)
    # skip one
    example = next(it)
    assert len(example) == 2
    assert example[0].shape == (2, 4, 100)
    assert example[1].shape == (2, 4)
    assert example[0][1, 2, :5].tolist() == [ord('d'), ord('e'), ord('f'), 0, 0]
    assert example[1].tolist() == [[1., 1., 0., 0.], [1., 1., 1., 1.]]

    os.remove(train_path)
    os.rmdir(temp_dir)

def test_squad_data():
    temp_dir = tempfile.mkdtemp()
    train_path = os.path.join(temp_dir, 'train.h5')
    with open(train_path, 'wb') as dst:
        print(base64.b64decode(TEST_SQUAD_BASE64_HDF5_DATA), file=dst)

    data = ExtractiveQAData(path=temp_dir, layout='squad')
    stream = data.get_stream('train', batch_size=3, shuffle=True, seed=3)
    assert set(stream.sources) == set(['contexts', 'contexts_mask',
                                       'questions', 'questions_mask',
                                       'answer_begins',
                                       'answer_ends'])
    batch = next(stream.get_epoch_iterator(as_dict=True))
    assert batch['contexts'].ndim == 2
    assert batch['contexts_mask'].ndim == 2
    assert batch['questions'].ndim == 2
    assert batch['questions_mask'].ndim == 2
    assert batch['answer_begins'].tolist() == [45, 78, 117]
    assert batch['answer_ends'].tolist() == [46, 80, 118]

    longest = batch['contexts_mask'].sum(axis=1).argmax()
    assert batch['contexts'][longest][-1] == data.vocab.eos

def test_random_span_scheme():
    scheme = RandomSpanScheme(10000, 100, 1)
    req_it = scheme.get_request_iterator()
    assert next(req_it) == slice(235, 335, None)

    dataset = IndexableDataset(['abc', 'def', 'xyz', 'ter'])
    scheme = RandomSpanScheme(4, 2, 1)
    stream = DataStream(dataset, iteration_scheme=scheme)
    it = stream.get_epoch_iterator()
    assert next(it) == (['def', 'xyz'],)


import collections

import arviz as az
import matplotlib.colors
import matplotlib.pyplot as plt
import numpy as np
from matplotlib.collections import LineCollection
from matplotlib.colors import LinearSegmentedColormap

# Create our own custom color maps to transition from black to the color of interest
cmap_C0 = LinearSegmentedColormap.from_list("blue", ["k", "C0"])
cmap_C1 = LinearSegmentedColormap.from_list("orange", ["k", "C1"])

def plot_cline(ax, x, y, dates, lw=1.0, primary=True):
    """
    Plot the orbit shaded by phase.

    Args:
        ax: matplotlib axes to plot onto
        x: the matplotlib x values (i.e., the Y astro coordinates)
        y: the matplotlib y values (i.e., the X astro coordinates)
        dates: the dates corresponding to each (x,y) value
        lw: the linewidth of the orbit
        primary: if True, plot in blue, else plot orange.

    Returns:
        None

    Example

            plot_cline(ax_orbit, Y_A, X_A, dates, primary=True)
            plot_cline(ax_orbit, Y_B, X_B, dates, primary=False)

    """

    cmap = cmap_C0 if primary else cmap_C1
    points = np.array([x, y]).T.reshape(-1, 1, 2)
    segments = np.concatenate([points[:-1], points[1:]], axis=1)

    lc = LineCollection(
        segments, cmap=cmap, norm=plt.Normalize(np.min(dates), np.max(dates))
    )
    lc.set_array(dates)
    lc.set_linewidth(lw)
    ax.add_collection(lc)

def plot_nodes(ax, Omega, radius):
    """
    Plot the line of nodes on the sky plot. The ascending node half is red, while the
    descending node half is blue.

    Args:
        ax: the matplotlib axes to plot onto
        Omega: the position angle of the ascending node (in radians)
        radius: how far away from the center to draw the line

    Returns:
        None
    """

    # calculate the slope of the line from Omega

    x_a = radius * np.sin(Omega)
    y_a = radius * np.cos(Omega)

    x_d = radius * np.sin(Omega + np.pi)
    y_d = radius * np.cos(Omega + np.pi)

    # plot in two segments
    ax.plot([0, x_a], [0, y_a], color="r")
    ax.plot([0, x_d], [0, y_d], color="b")

def plot_errorbar(ax, thetas, rhos, theta_errs, rho_errs, **kwargs):
    """
    Custom routine for plotting polar error bars onto cartesian plane
    All kwargs sent to plot. Thetas are in radians.
    """

    thetas = np.atleast_1d(thetas)
    rhos = np.atleast_1d(rhos)
    theta_errs = np.atleast_1d(theta_errs)
    rho_errs = np.atleast_1d(rho_errs)

    assert len(thetas) == len(rhos) == len(theta_errs) == len(rho_errs)

    for theta, rho, theta_err, rho_err in zip(thetas, rhos, theta_errs, rho_errs):

        # In this notation, x has been flipped.
        # x = rho * np.sin(theta)
        # y = rho * np.cos(theta)

        # Calculate two lines, one for theta_err, one for rho_err

        # theta error
        # theta error needs a small correction in the rho direction in order to go right through the data point
        delta = rho * (1 / np.cos(theta_err) - 1)
        xs = (rho + delta) * np.sin(np.array([theta - theta_err, theta + theta_err]))
        ys = (rho + delta) * np.cos(np.array([theta - theta_err, theta + theta_err]))
        ax.plot(xs, ys, **kwargs)

        # rho error
        xs = np.sin(theta) * np.array([(rho - rho_err), (rho + rho_err)])
        ys = np.cos(theta) * np.array([(rho - rho_err), (rho + rho_err)])
        ax.plot(xs, ys, **kwargs)

def efficient_trace(trace, var_names, figstem, max_panel=6):
    """
    Wrap the arviz call for trace to speed things up by splitting into smaller figures bundled as a scrollable PDF.
    """

    N = len(var_names)
    n = 0
    i = 0
    while n <= N:
        print("Plotting {:} of {:}".format(n, N))
        plot_vars = var_names[n : n + max_panel]

        az.plot_trace(trace, var_names=plot_vars)
        plt.savefig(figstem.format(i))
        plt.close("all")
        n += max_panel
        i += 1

def efficient_autocorr(trace, var_names, figstem, max_panel=6):
    """
    Wrap the arviz call for correlation to speed things up by splitting into smaller figures bundled as a scrollable PDF.
    """

    N = len(var_names)
    n = 0
    i = 0
    while n <= N:
        plot_vars = var_names[n : n + max_panel]

        az.plot_autocorr(trace, var_names=plot_vars)
        plt.savefig(figstem.format(i))
        plt.close("all")
        n += max_panel
        i += 1


from django.urls import path, include
from rest_framework import routers

from .views import StoryViewSet, EpisodeViewSet

router = routers.DefaultRouter()

router.register(r'episodes', EpisodeViewSet, basename='episodes')
router.register(r'(?P<story_id>[0-9]+)/episode', EpisodeViewSet, basename='story_episodes')
router.register(r'', StoryViewSet, basename='stories')

urlpatterns = [
    path('', include(router.urls)),
]


from mesa import Agent
from collections import deque
import random

class Car(Agent):
    """
    Agent that moves randomly.
    Attributes:
        unique_id: Agent's ID
        direction: Randomly chosen direction chosen from one of eight directions
    """
    def __init__(self, unique_id, model, pos):
        """
        Creates a new random agent.
        Args:
            unique_id: The agent's ID
            model: Model reference for the agent
        """
        super().__init__(unique_id, model)
         #Top Rigth Down Left
        self.pos = pos
        self.steps_taken = 0
        self.front = (self.pos[0]-1,self.pos[1])
        self.randPoint = random.choice(self.model.endPointsM)
        self.path = self.BFS()
        self.stepsBFS = 0

    def BFS(self):

        """ BFS BEARBONES"""
        typeArray = []
        prev = []
        path = []
        rPoint = self.randPoint

        for i in range(self.model.grid.height):
            rowList = []
            prevList = []
            for j in range(self.model.grid.width):
                if(len(self.model.grid[i, j]) > 0):
                    rowList.append(self.model.grid[i, j][0])
                    prevList.append(0)
                else:
                    rowList.append(self.model.grid[i, j])
                    prevList.append(0)
            prev.append(prevList)
            typeArray.append(rowList)
        # To move left, right, up and down, and diagonally
        delta_x = [-1, 1, 0, 0, 1, -1, -1, 1]
        delta_y = [0, 0, 1, -1, -1, 1, -1, 1]

        def valid(x, y):
            if x < 0 or x >= len(typeArray) or y < 0 or y >= len(typeArray[x]):
                return False
            return (not isinstance(typeArray[x][y], Obstacle))

        def solve(start, end):
            Q = deque([start])
            dist = {start: 0}
            while len(Q):
                curPoint = Q.popleft() #Move to one of the neighbors, remove visited
                curDist = dist[curPoint] #Gets the nums of levels it took to reach this node
                if curPoint == end: #If we have found a mathc to the endpoint
                    path.append(curPoint)
                    return prev, curDist

                for dx, dy in zip(delta_x, delta_y):
                    nextPoint = (0,0)
                    if isinstance(typeArray[curPoint[0]][curPoint[1]], Road):
                        dir = (typeArray[curPoint[0]][curPoint[1]]).direction
                        if dir == "Up" and dy == 1:
                            nextPoint = (curPoint[0] + dx, curPoint[1] + dy)
                        elif dir == "Down" and dy == -1:
                            nextPoint = (curPoint[0] + dx, curPoint[1] + dy)
                        elif dir == "Right" and dx == 1:
                            nextPoint = (curPoint[0] + dx, curPoint[1] + dy)
                        elif dir == "Left" and dx == -1:
                            nextPoint = (curPoint[0] + dx, curPoint[1] + dy)
                    elif isinstance(typeArray[curPoint[0]][curPoint[1]], Destination):
                        nextPoint = (curPoint[0] + dx, curPoint[1] + dy)
                        break

                    if not valid(nextPoint[0], nextPoint[1]) or nextPoint in dist.keys(): #Node within boundaries, and not of type obstacle and not visited
                        continue
                    dist[nextPoint] = curDist + 1

                    Q.append(nextPoint) #Add the next unvisited node
                    prev[nextPoint[0]][nextPoint[1]] = curPoint #Keep track of the parent node of the next node
                    path.append(curPoint)

        def reconstructPath(s, e, prev):
            #Rebuild the path and invert it
            path = []
            at = e
            for i in range(len(prev[0])):
                while at != 0:
                    at = prev[0][at[0]][at[1]]
                    path.append(at)

            path.reverse()

            path.pop(0) #Remove garbage data (if i remove the code, the BFS will break)
            path.append(e) #Add the goal at the end of the array

            if path[0] == s:
                return path
            else:
                return []

        return reconstructPath(self.pos, rPoint, solve(self.pos, rPoint))

    def randomPoint(self):
        """ Get End Points """
        endPointsM = self.model.endPointsM
        return(random.choice(endPointsM))

    def newFront(self, road):
        for agentR in road:
            if isinstance(agentR, Road):
                if agentR.direction == "Right":
                    self.front = (self.pos[0]+1,self.pos[1])
                elif agentR.direction == "Left":
                    self.front = (self.pos[0]-1,self.pos[1])
                elif agentR.direction == "Down":
                    self.front = (self.pos[0],self.pos[1]-1)
                elif agentR.direction == "Up":
                    self.front = (self.pos[0],self.pos[1]+1)

    def move(self):
        """ Get End Points """
        possible_end_points = self.model.grid.get_neighborhood(
            self.pos,
            moore=True, # Boolean for whether to use Moore neighborhood (including diagonals) or Von Neumann (only up/down/left/right).
            include_center=True)

        """Movimineto Ejec"""
        finalPath = self.path
        road = self.model.grid.get_cell_list_contents(self.pos)
        self.newFront(road)
        trafLight = self.model.grid.get_cell_list_contents(self.front)

        #If the traffic light is red, dont move else move
        if self.pos != self.randPoint:
            for agentL in trafLight:
                if isinstance(agentL, Traffic_Light):
                    if not agentL.state:
                        return
                elif isinstance(agentL, Car):
                    return
            #If next tile in the path has a car on top of it, re caulculate the BFS in order to avoid it
            if(isinstance(finalPath[self.stepsBFS], Car)):
                self.path = self.BFS()
            else:
                self.stepsBFS += 1
                self.model.grid.move_agent(self, finalPath[self.stepsBFS])
        else:
            self.model.grid.move_agent(self, possible_end_points[possible_end_points.index(self.randPoint)])
            newpos = random.choice(self.model.roadList)
            if(isinstance(newpos, Car)):
                newpos = random.choice(self.model.roadList)
            self.model.grid.move_agent(self, newpos)
            self.steps_taken = 0
            self.front = (self.pos[0]-1,self.pos[1])
            self.randPoint = random.choice(self.model.endPointsM)
            self.path = self.BFS()
            self.stepsBFS = 0

        """ BFS BEARBONES """

    def step(self):
        """
        Determines the new direction it will take, and then moves
        """

        self.move()

class Traffic_Light(Agent):
    """
    Traffic light. Where the traffic lights are in the grid.
    """
    def __init__(self, unique_id, model, state = False, timeToChange = 10):
        super().__init__(unique_id, model)
        """
        Creates a new Traffic light.
        Args:
            unique_id: The agent's ID
            model: Model reference for the agent
            state: Whether the traffic light is green or red
            timeToChange: After how many step should the traffic light change color
        """
        self.state = state
        self.timeToChange = timeToChange

    def step(self):
        """
        To change the state (green or red) of the traffic light in case you consider the time to change of each traffic light.
        """
        pass

class Destination(Agent):
    """
    Destination agent. Where each car should go.
    """
    def __init__(self, unique_id, model):
        super().__init__(unique_id, model)

    def step(self):
        pass

class Obstacle(Agent):
    """
    Obstacle agent. Just to add obstacles to the grid.
    """
    def __init__(self, unique_id, model):
        super().__init__(unique_id, model)

    def step(self):
        pass

class Road(Agent):
    """
    Road agent. Determines where the cars can move, and in which direction.
    """
    def __init__(self, unique_id, model, direction= "Left"):
        """
        Creates a new road.
        Args:
            unique_id: The agent's ID
            model: Model reference for the agent
            direction: Direction where the cars can move
        """
        super().__init__(unique_id, model)
        self.direction = direction

    def step(self):
        pass


import asyncio
from time import time
from typing import Any, Awaitable, Callable
from unittest import mock
from uuid import uuid4

import pytest_asyncio
from aiokafka import AIOKafkaProducer
from httpx import AsyncClient
from motor.core import AgnosticClient
from motor.motor_asyncio import AsyncIOMotorClient
from src.common.authorization import JwtClaims, JwtUserSchema
from src.common.databases import get_kafka_producer, get_mongodb
from src.main import app
from src.settings.app import get_app_settings

settings = get_app_settings()

GetRequestType = Callable[
    [str, dict[str, Any]],
    Awaitable[tuple[dict, int]],
]

@pytest_asyncio.fixture(scope="session")
def event_loop():
    policy = asyncio.get_event_loop_policy()
    loop = policy.new_event_loop()
    yield loop
    loop.close()

@pytest_asyncio.fixture(scope="function")
async def db_session():
    mongodb = AsyncIOMotorClient(settings.mongo.dsn)
    producer = mock.AsyncMock()

    def get_test_mongodb() -> AgnosticClient:
        if mongodb is None:
            raise RuntimeError("MongoDB client has not been defined.")

        return mongodb

    def get_test_kafka_producer() -> AIOKafkaProducer:
        if producer is None:
            raise RuntimeError("Kafka producer has not been defined.")

        return producer

    app.dependency_overrides[get_mongodb] = get_test_mongodb
    app.dependency_overrides[get_kafka_producer] = get_test_kafka_producer

    yield mongodb

    collections = await mongodb[settings.mongo.db_name].list_collection_names()
    for collection in collections:
        await mongodb[settings.mongo.db_name].drop_collection(collection)

@pytest_asyncio.fixture(scope="function")
async def client():
    base_url = f"http://{settings.service.host}:{settings.service.port}/api/v1"
    async with AsyncClient(app=app, base_url=base_url) as client:
        yield client

@pytest_asyncio.fixture(scope="session", autouse=True)
async def mock_fast_api_limiter_init():
    """Mock FastAPILimiter.init() method to prevent 429 status code during tests."""
    with mock.patch(
        "fastapi_limiter.depends.FastAPILimiter", new_callable=mock.AsyncMock
    ):
        yield

@pytest_asyncio.fixture(scope="session", autouse=True)
async def mock_jwt():
    jwt_user = JwtUserSchema(
        id=str(uuid4()),
        permissions=["test"],
    )
    jwt_claim = JwtClaims(
        user=jwt_user,
        iat=1,
        access_jti="test",
        refresh_jti="test",
        type="test",
        exp=int(time()) + 60,
    )

    with mock.patch("src.common.authorization.decode_token") as mock_jwt:
        mock_jwt.return_value = jwt_claim
        yield jwt_claim


"""
citations
"""

import logging

import pickleshare
import requests
from pathlib import Path

log = logging.getLogger(__name__)

# silence request logging
logging.getLogger("requests.packages.urllib3.connectionpool").setLevel(
    logging.WARNING)

def get_cache(cache_dir):
    # The reason to use pickleshare is that it uses a separate pickle file
    # for each key (i.e. doi), so if the process crashes, at least most of the
    # looked up citations/metadata is saved to file
    cache_path = Path(cache_dir)
    cache_path.mkdir(parents=True, exist_ok=True)
    log.info('reading cached data from ' + cache_dir)
    cache = pickleshare.PickleShareDB(cache_dir)
    return cache

def get_citation(doi, cache,
                 style='chicago-fullnote-bibliography',
                 strip_doi=True, online=True):
    """
    Get formatted citation string for DOI from CrossRef

    Parameters
    ----------
    doi
    cache
    style
    strip_doi
    online

    Returns
    -------

    """
    try:
        return cache[doi]
    except KeyError:
        pass

    if not online:
        log.warning('skipping online lookup of citation for DOI {}'.format(doi))
        return ''

    headers = {'Accept': 'text/bibliography; style={}'.format(style)}
    attempts = 10
    response = None

    for i in range(attempts):
        response = requests.get('http://dx.doi.org/{}'.format(doi), headers=headers)
        if response.ok:
            break
    else:
        if response:
            log.error('request for formated citation of {} returned {}: {}'.format(
                doi, response.status_code, response.reason))
        else:
            log.error('request for formated citation of {} failed'.format(doi))
        return ''

    log.info('request for formated citation of {} succeeded'.format(doi))

    citation = response.content.decode('utf-8')

    if strip_doi:
        # TODO 3: won't work for other styles
        citation = citation.split(' doi:')[0]

    cache[doi] = citation
    return citation

def get_all_metadata(doi, cache, online=True):
    """
    Get metadata for DOI, including metadata from ISSN
    """
    metadata = get_doi_metadata(doi, cache, online)
    issn = metadata['ISSN']
    if issn:
        issn_metadata = get_issn_metadata(issn, cache, online)
        metadata.update(issn_metadata)
    return metadata

def get_doi_metadata(doi, cache, online=True):
    """
    Get metadata for DOI
    """
    doi_metadata = request_doi_metadata(doi, cache, online=online)
    metadata = {'doi': doi}

    for key in 'title', 'publisher':
        try:
            metadata[key] = doi_metadata[key]
        except KeyError:
            metadata[key] = None
            log.warning('no {} found for DOI {}'.format(key, doi))

    try:
        metadata['ISSN'] = doi_metadata['ISSN'][0]
    except (KeyError, IndexError):
        metadata['ISSN'] = None
        log.warning('no ISSN found for DOI {}'.format(doi))

    try:
        metadata['journal'] = doi_metadata['container-title']
    except KeyError:
        metadata['journal'] = None
        log.warning('no journal (container-title) found for DOI {}'.format(doi))

    # example fragment:
    # 'published-online': {'date-parts': [[2009, 8, 30]]},
    # 'published-print': {'date-parts': [[1998, 1, 22]]}
    published = (doi_metadata.get('published-online') or
                 doi_metadata.get('published-print') or
                 doi_metadata.get('issued'))

    try:
        parts = published['date-parts'][0]
    except (TypeError, KeyError, IndexError):
        parts = []

    try:
        metadata['year'] = parts[0]
    except IndexError:
        metadata['year'] = None
        log.warning('no publication date found for DOI {}'.format(doi))

    try:
        metadata['month'] = parts[1]
    except IndexError:
        metadata['month'] = None

    try:
        metadata['day'] = parts[2]
    except IndexError:
        metadata['day'] = None

    return metadata

def request_doi_metadata(doi, cache, attempts=10, online=True):
    """
    Request metadata for DOI from CrossRef
    """
    try:
        return cache[doi]
    except KeyError:
        pass

    if not online:
        log.warning('skipping online lookup for DOI {}'.format(doi))
        return {}

    headers = {'Accept': 'application/vnd.citationstyles.csl+json'}
    response = None

    for i in range(attempts):
        response = requests.get('http://dx.doi.org/{}'.format(doi),
                                headers=headers)
        if response.ok:
            break
    else:
        if response:
            log.error('request for metadata of DOI {} returned {}: {}'.format(
                doi, response.status_code, response.reason))
        else:
            log.error('request for metadata of DOI {} failed'.format(doi))
        return {}

    log.info('request for metadata of DOI {} succeeded'.format(doi))
    metadata = response.json()
    cache[doi] = metadata
    return metadata

def get_issn_metadata(issn, cache, online=True):
    """
    Get metadata for ISSN
    """
    issn_metadata = request_issn_metadata(issn, cache, online=online)
    metadata = {}

    try:
        metadata['journal'] = issn_metadata['title']
    except KeyError:
        pass

    try:
        metadata['publisher'] = issn_metadata['publisher']
    except KeyError:
        pass

    return metadata

def request_issn_metadata(issn, cache, attempts=10, online=True):
    """
    Request metadata for ISSN from CrossRef
    """
    try:
        return cache[issn]
    except KeyError:
        pass

    if not online:
        log.warning('skipping online lookup for ISSN {}'.format(issn))
        return {}

    response = None

    for i in range(attempts):
        response = requests.get(
            'http://api.crossref.org/journals/{}'.format(issn))
        if response.ok:
            break
    else:
        if response:
            log.error('request for metadata of ISSN {} returned {}: {}'.format(
                issn, response.status_code, response.reason))
        else:
            log.error('request for metadata of ISSN {} failed'.format(issn))

        return {}

    log.info('request for metadata of ISSN {} succeeded'.format(issn))
    message = response.json()['message']
    cache[issn] = message
    return message

def clean_metadata_cache(cache_dir):
    """
    Remove records with None values from metadata cache

    This means that on the next call to add_metadata(),
    new metadata will be requested for the removed records.
    """
    log.info('cleaning cached metadata from ' + cache_dir)
    cache = pickleshare.PickleShareDB(cache_dir)
    to_delete = []

    for key in cache.keys():
        func = get_doi_metadata if '/' in key else get_issn_metadata
        metadata = func(key, cache)
        if None in metadata.values():
            to_delete.append(key)

    for key in to_delete:
        log.info('removing incomplete cached metadata for key {}'.format(key))
        del cache[key]


import numpy as np
import matplotlib.pyplot as plt
from matplotlib.patches import Polygon

A = np.array([[1,0.1,0.1],
              [0.1,1,0.9],
              [0.1,0.9,1]])

def plot_mu(mus,*args, triag = False, **kwargs):
    mus = mus/np.sum(mus, axis = 0)
    x = mus[1] + mus[-1]/2
    y = mus[-1]
    if triag:
        x = np.append(x, x[0])
        y = np.append(y, y[0])
    plt.plot(x,y, *args, **kwargs)
    return x,y

fig = plt.figure()
ax = plt.gca()
ax.set_xticks([])
ax.set_yticks([])

ax.set_frame_on(False)
plot_mu(np.eye(3), 'k', triag = True)
x,y = plot_mu(A, 'ko')

coex = Polygon(np.array([x,y]).T, closed = True,
               alpha = 0.5, color = "g")
ax.add_patch(coex)

rc = 1/len(A)*(np.sum(A/np.linalg.norm(A, axis = 1), axis = 0))
plot_mu(rc, 'ro')

itera = 5000
mu1 = np.linspace(0,1,itera)
mu2 = np.linspace(0,1-mu1,itera)
mu1 = np.repeat([mu1], itera, axis = 0)
mu3 = 1 - mu1 - mu2
mu = np.array([mu1, mu2, mu3])
good = np.abs(np.einsum("i, inm->nm", rc, mu)/np.linalg.norm(mu, axis = 0)-0.75)<1e-4

plot_mu(mu[:,good], ',b')
mu_theta = mu[:,good]
pos_equi = np.all(np.linalg.solve(A, mu_theta)>0, axis = 0)
plot_mu(mu[:,good][:,pos_equi], ',', color = "orange")

ax.text(0,-0.05,"$\mu_1$", ha = "center")
ax.text(1,-0.05,"$\mu_2$", ha = "center")
ax.text(0.5,1+0.05,"$\mu_3$", ha = "center")

fig.savefig("Figure_S7.pdf")


import logging
import os
import unittest
import fudge
from google.appengine.ext import db
from crauth import users, models, permissions

os.environ['SERVER_NAME'] = 'localhost'
os.environ['SERVER_PORT'] = '8000'
os.environ['USER_EMAIL'] = 'test@example.com'
os.environ['USER_IS_ADMIN'] = '1'

FAKE_DOMAIN = 'fakedomain.com'
FAKE_EMAIL = 'fake@%s' % FAKE_DOMAIN

class PermissionsTestCase(unittest.TestCase):
    def setUp(self):
        if os.environ.has_key(users._ENVIRON_EMAIL):
            del os.environ[users._ENVIRON_EMAIL]
        if os.environ.has_key(users._ENVIRON_DOMAIN):
            del os.environ[users._ENVIRON_DOMAIN]
        domain = models.AppsDomain.get_or_insert(
            key_name=FAKE_DOMAIN,
            domain=FAKE_DOMAIN,
        )
        db.delete(models.UserPermissions.all().fetch(100))
        db.delete(models.Role.all().fetch(100))

    def set_testing_user(self):
        users._set_testing_user(FAKE_EMAIL, FAKE_DOMAIN)
        return users.get_current_user()

    def fake_is_admin(self, is_admin=True):
        fake_is_admin = fudge.Fake('users.User.is_admin',
                                   callable=True)
        fake_is_admin.returns(is_admin)
        return fudge.patched_context(users.User, 'is_admin',
                                     fake_is_admin)

    def testFakeIsAdmin(self):
        self.set_testing_user()
        with self.fake_is_admin(True):
            self.assertTrue(users.is_current_user_admin())
        with self.fake_is_admin(False):
            self.assertFalse(users.is_current_user_admin())

    def testHasPerm(self):
        user = self.set_testing_user()
        permissions = models.UserPermissions(
            key_name=user._email,
            user_email=user._email,
            permissions=['test_perm1', 'test_perm2'],
        )
        permissions.put()
        with self.fake_is_admin(False):
            self.assertTrue(user.has_perms(['test_perm1', 'test_perm2']))

    def testHasPerms(self):
        user = self.set_testing_user()
        permissions = models.UserPermissions(
            key_name=user._email,
            user_email=user._email,
            permissions=['test_perm'],
        )
        permissions.put()
        with self.fake_is_admin(False):
            self.assertTrue(user.has_perm('test_perm'))

    def testDoesntHavePerm(self):
        user = self.set_testing_user()
        permissions = models.UserPermissions(
            key_name=user._email,
            user_email=user._email,
            permissions=['some_perm'],
        )
        permissions.put()
        with self.fake_is_admin(False):
            self.assertFalse(user.has_perm('some_other_perm'))

    def testAdminHasAllPerms(self):
        user = self.set_testing_user()
        permissions = models.UserPermissions(
            key_name=user._email,
            user_email=user._email,
            permissions=['some_perm'],
        )
        permissions.put()
        with self.fake_is_admin(True):
            self.assertTrue(user.has_perm('some_perm'))
            self.assertTrue(user.has_perm('some_other_perm'))
            self.assertTrue(user.has_perms(['some_perm', 'some_other_perm']))

    def testDoesntHaveAllPerms(self):
        user = self.set_testing_user()
        permissions = models.UserPermissions(
            key_name=user._email,
            user_email=user._email,
            permissions=['some_perm'],
        )
        permissions.put()
        with self.fake_is_admin(False):
            self.assertFalse(user.has_perms(['some_perm', 'some_other_perm']))

    def testHasAdminPerm(self):
        user = self.set_testing_user()
        with self.fake_is_admin(True):
            for perm in permissions.ADMIN_PERMS:
                self.assertTrue(user.has_perm(perm))
            self.assertTrue(user.has_perms(permissions.ADMIN_PERMS))

    def testDoesntHaveAdminPerm(self):
        user = self.set_testing_user()
        permissions = models.UserPermissions(
            key_name=user._email,
            user_email=user._email,
            permissions=['admin'],
        )
        permissions.put()
        with self.fake_is_admin(False):
            self.assertFalse(user.has_perm('admin'))

    def testRoleForDomain(self):
        domain_1 = models.AppsDomain(domain='domain1')
        domain_1.put()
        domain_2 = models.AppsDomain(domain='domain2')
        domain_2.put()
        role = models.Role(
            name='Test Role',
            domain=domain_1,
        )
        role.put()
        role = models.Role(
            name='Test Role',
            domain=domain_2,
        )
        role.put()
        self.assertEqual(len(models.Role.for_domain(domain_1).fetch(2)), 1)

    def testHasPermThroughRole(self):
        user = self.set_testing_user()
        role = models.Role(
            name='Test Role',
            domain=user.domain(),
            permissions=['some_perm'],
        )
        role.put()
        permissions = models.UserPermissions(
            key_name=user._email,
            user_email=user._email,
            roles=[role.key()],
        )
        permissions.put()
        with self.fake_is_admin(False):
            self.assertTrue(user.has_perm('some_perm'))


import sys,tweepy,csv,re
from textblob import TextBlob
import matplotlib.pyplot as plt

class SentimentAnalysis:

    def __init__(self):
        self.tweets = []
        self.tweetText = []

    def DownloadData(self):
        # authenticating
        consumerKey = ""
        consumerSecret = ""
        accessToken = ""
        accessTokenSecret = ""
        auth = tweepy.OAuthHandler(consumerKey, consumerSecret)
        auth.set_access_token(accessToken, accessTokenSecret)
        api = tweepy.API(auth)

        # input for term to be searched and how many tweets to search
        searchTerm = "#scotrail"
        NoOfTerms = 100

        # searching for tweets
        self.tweets = tweepy.Cursor(api.search, q=searchTerm, lang = "en").items(NoOfTerms)

        # creating some variables to store info
        polarity = 0
        positive = 0
        negative = 0
        neutral = 0

        # iterating through tweets fetched
        for tweet in self.tweets:
            #Append to temp so that we can store in csv later. I use encode UTF-8
            self.tweetText.append(self.cleanTweet(tweet.text).encode('utf-8'))
            # print (tweet.text.translate(non_bmp_map))    #print tweet's text
            analysis = TextBlob(tweet.text)
            # print(analysis.sentiment)  # print tweet's polarity
            polarity += analysis.sentiment.polarity  # adding up polarities to find the average later

            if (analysis.sentiment.polarity == 0.00): # adding reaction of how people are reacting to find average later
                neutral += 1
            elif (analysis.sentiment.polarity > 0.00):
                positive += 1
            elif (analysis.sentiment.polarity < 0.00):
                negative += 1

        # finding average of how people are reacting
        positive = self.percentage(positive, NoOfTerms)
        negative = self.percentage(negative, NoOfTerms)
        neutral = self.percentage(neutral, NoOfTerms)

        # finding average reaction
        polarity = polarity / NoOfTerms

    with open('Tweet_Sentiments.csv', 'w') as scorefile:
        scoreFileWriter = csv.writer(scorefile)
        # printing out data
        print("How people are reacting to " + searchTerm + " by analyzing " + str(NoOfTerms) + " tweets.")
        print()
        print("General Report: ")
        scoreFileWriter.writerow([tweet.text, analysis.sentiment.polarity])

        if (polarity == 0):
            print("Neutral")
        elif (polarity > 0.00):
            print("Positive")
        elif (polarity < 0.00):
            print("Negative")

        print()
        print("Detailed Report: ")
        print(str(positive) + "% people were positive")
        print(str(negative) + "% people were negative")
        print(str(neutral) + "% people were neutral")

        self.plotPieChart(positive, negative, neutral, searchTerm, NoOfTerms)

    def cleanTweet(self, tweet):
        # Remove Links, Special Characters etc from tweet
        return ' '.join(re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t]) | (\w +:\ / \ / \S +)", " ", tweet).split())

    # function to calculate percentage
    def percentage(self, part, whole):
        temp = 100 * float(part) / float(whole)
        return format(temp, '.2f')

    def plotPieChart(self, positive, negative, neutral, searchTerm, noOfSearchTerms):
        labels = ['Positive [' + str(positive) + '%]', 'Neutral [' + str(neutral) + '%]', 'Negative [' + str(negative) + '%]']
        sizes = [positive, neutral, negative]
        colors = ['yellowgreen','gold', 'red',]
        patches, texts = plt.pie(sizes, colors=colors, startangle=90)
        plt.legend(patches, labels, loc="best")
        plt.title('How people are reacting to ' + searchTerm + ' by analyzing ' + str(noOfSearchTerms) + ' Tweets.')
        plt.axis('equal')
        plt.tight_layout()
        plt.show()

if __name__== "__main__":
    sa = SentimentAnalysis()
    sa.DownloadData()


import pandas as pd
import numpy as np
import lightgbm as lgb
import matplotlib.pyplot as plt
import category_encoders as ce
import optuna  # ハイパーパラメータチューニング自動化ライブラリ
from tqdm import tqdm
from optuna.integration import lightgbm_tuner
from python_file.kaggle.common import common_funcs as cf
from sklearn.model_selection import train_test_split, StratifiedKFold
from sklearn.metrics import accuracy_score
from sklearn.feature_selection import RFECV, RFE
from python_file.kaggle.predict_winner.common import Common

DATA_DIR = "src/sample_data/Kaggle/predict_winner"
TRAIN_PATH = f"{DATA_DIR}/train_data.csv"
TEST_PATH = f"{DATA_DIR}/test_data.csv"
BUKI_PATH = f"{DATA_DIR}/weapon.csv"
REMOVAL_COLS = ["lobby", "game-ver"]

WEAPON_MAP = {
    "heroblaster_replica": "hotblaster",
    "herobrush_replica": "hokusai",
    "herocharger_replica": "splatcharger",
    "heromaneuver_replica": "maneuver",
    "heroroller_replica": "splatroller",
    "heroshelter_replica": "parashelter",
    "heroshooter_replica": "sshooter",
    "heroslosher_replica": "bucketslosher",
    "herospinner_replica": "splatspinner",
    "octoshooter_replica": "sshooter",
}

RANK_MAP = {
    "c-": 1,
    "c": 2,
    "c+": 3,
    "b-": 4,
    "b": 5,
    "b+": 6,
    "a-": 7,
    "a": 8,
    "a+": 9,
    "s-": 10,
    "s": 11,
    "s+": 12,
    "x": 13,
}

cm = Common()

def get_best_params(train_x, train_y):
    tr_x, val_x, tr_y, val_y = train_test_split(
        train_x, train_y, test_size=0.2, random_state=1
    )
    tr_set = lgb.Dataset(tr_x, tr_y)
    val_set = lgb.Dataset(val_x, val_y, reference=tr_set)
    best_params = {}
    params = {
        "objective": "binary",
        "metric": "binary_logloss",
        "boosting_type": "gbdt",
    }
    best_params = {}
    tuning_history = []
    gbm = lightgbm_tuner.train(
        params,
        tr_set,
        valid_sets=val_set,
        num_boost_round=1000,
        early_stopping_rounds=50,
        verbose_eval=10,
        best_params=best_params,
        tuning_history=tuning_history,
    )
    return best_params

def objective(X, y, trial):
    """最適化する目的関数"""
    tr_x, val_x, tr_y, val_y = train_test_split(X, y, test_size=0.2, random_state=1)
    gbm = lgb.LGBMClassifier(
        objective="binary",
        boosting_type="gbdt",
        n_jobs=4,
        n_estimators=1000,
        early_stopping_rounds=100,
        learning_rate=0.01,
    )
    # RFE で取り出す特徴量の数を最適化する
    n_features_to_select = (
        trial.suggest_int("n_features_to_select", 100, len(list(tr_x.columns))),
    )
    rfe = RFE(estimator=gbm, n_features_to_select=n_features_to_select)
    rfe.fit(tr_x, tr_y)
    selected_cols = list(tr_x.columns[rfe.support_])

    tr_x_selected = tr_x.loc[:, selected_cols]
    val_x_selected = val_x.loc[:, selected_cols]
    gbm.fit(
        tr_x_selected,
        tr_y,
        eval_set=[(val_x_selected, val_y)],
        early_stopping_rounds=20,
    )
    y_pred = gbm.predict(val_x_selected)
    accuracy = accuracy_score(val_y, y_pred)
    return accuracy

def train(train_x, train_y, kfold, best_params=None):
    params = {
        "objective": "binary",
        "boosting_type": "gbdt",
        "metric": {"binary_logloss"},
        "num_leaves": 50,
        "min_data_in_leaf": 100,
        "learning_rate": 0.1,
        "feature_fraction": 0.7,
    }
    models = []
    for i, (tr_idx, val_idx) in enumerate(kfold.split(train_x, train_y)):
        tr_x = train_x.iloc[tr_idx].reset_index(drop=True)
        tr_y = train_y.iloc[tr_idx].reset_index(drop=True)
        val_x = train_x.iloc[val_idx].reset_index(drop=True)
        val_y = train_y.iloc[val_idx].reset_index(drop=True)

        tr_set = lgb.Dataset(tr_x, tr_y)
        val_set = lgb.Dataset(val_x, val_y, reference=tr_set)

        evals_result = {}
        model = lgb.train(
            params=params,
            train_set=tr_set,
            valid_sets=[val_set, tr_set],
            num_boost_round=10000,
            early_stopping_rounds=100,
            verbose_eval=1,
            evals_result=evals_result,
            feval=accuracy,
        )

        importance = pd.DataFrame(
            model.feature_importance(), index=train_x.columns, columns=["importance"]
        ).sort_values("importance", ascending=[False])

        # print(f"######################importance#####################")
        # print(importance.head(50))

        # 検証結果の描画
        fig = lgb.plot_metric(evals_result)
        plt.savefig(f"{DATA_DIR}/learning_curve_{i+1}.png")

        models.append(model)

    return models

def accuracy(preds, data, threshold=0.5):
    """精度 (Accuracy) を計算する関数"""
    weight = data.get_weight()
    true_label = data.get_label()
    pred_label = (preds > threshold).astype(int)
    acc = np.average(true_label == pred_label, weights=weight)
    return "accuracy", acc, True

def predict(model, test_x, threshold):
    y_pred = model.predict(test_x, num_iteration=model.best_iteration)
    pred = [0 if i < threshold else 1 for i in y_pred]
    return pd.Series(pred)

def run_all():
    train_raw_data = pd.read_csv(TRAIN_PATH)
    test_raw_data = pd.read_csv(TEST_PATH)
    buki_raw_data = pd.read_csv(BUKI_PATH)

    train_raw_data = train_raw_data.replace(WEAPON_MAP)
    test_raw_data = test_raw_data.replace(WEAPON_MAP)
    buki_raw_data = buki_raw_data.replace(WEAPON_MAP)

    test_raw_data["y"] = 0
    train_raw_data["usage"] = 0  # for train
    test_raw_data["usage"] = 1  # for test

    X = cm.make_input_output(train_raw_data, with_y=False)
    train_data = pd.concat([train_raw_data, X], axis=1)

    # 武器ごとの勝率計算
    win_rate_df = []
    for buki in buki_raw_data.key.unique():
        # print(buki, win_rate(buki))
        rate, count = cm.win_rate(buki, train_data)
        win_rate_df.append([buki, rate, count])
    win_rate_df = pd.DataFrame(win_rate_df, columns=["buki", "win_rate", "count"])
    win_rate_df = win_rate_df.sort_values(by="win_rate", ascending=False)
    buki_ja_dict = buki_raw_data[["key", "reskin"]].set_index("key").to_dict()["reskin"]
    win_rate_df.buki = win_rate_df.buki.map(buki_ja_dict)
    win_rate_dict = (
        win_rate_df.reset_index(drop=True)
        .loc[:, ["buki", "win_rate"]]
        .set_index("buki")
        .to_dict()
    )[
        "win_rate"
    ]  # 全体の勝率dict(武器名: 勝率)

    # モード毎の武器勝率計算
    win_rate_mode_df = pd.DataFrame()
    for mode in train_data["mode"].unique():
        win_rate_df = []
        train_mode = train_data[train_data["mode"] == mode]
        for buki in buki_raw_data.key.unique():
            # print(buki, win_rate(buki))
            rate, count = cm.win_rate(buki, train_mode)
            win_rate_df.append([buki, rate, count])
        win_rate_df = pd.DataFrame(
            win_rate_df, columns=[mode + "_buki", mode + "_win_rate", mode + "_count"]
        )
        win_rate_df = win_rate_df.sort_values(by=mode + "_win_rate", ascending=False)
        buki_ja_dict = (
            buki_raw_data[["key", "reskin"]].set_index("key").to_dict()["reskin"]
        )
        win_rate_df[mode + "_buki"] = win_rate_df[mode + "_buki"].map(buki_ja_dict)
        win_rate_df = win_rate_df.reset_index(drop=True)
        win_rate_mode_df = pd.concat([win_rate_mode_df, win_rate_df], axis=1)

    # 武器と射程距離の関係分類計算
    buki_range_distance_dict = (
        buki_raw_data.loc[:, ["category2", "key"]]
        .set_index("key")
        .to_dict()["category2"]
    )

    train_data, test_data = cm.make_feature(train_raw_data, test_raw_data)
    raw_data = pd.concat([train_data, test_data], axis=0).reset_index(drop=True)
    raw_data = raw_data.fillna(raw_data.mode().iloc[0])

    data = cm.add_count_range_distance(raw_data, buki_range_distance_dict)
    # data = cm.calc_weapons_win_rate_avg(data, win_rate_dict)
    # data = cm.calc_weapons_win_rate_avg_per_mode(data, win_rate_mode_df)

    categorical_columns = [x for x in data.columns if data[x].dtype == "object"]

    ce_oe = ce.OrdinalEncoder(cols=categorical_columns, handle_unknown="impute")
    encorded_data = ce_oe.fit_transform(data)

    train_data = (
        encorded_data[encorded_data["usage"] == 0]
        .drop(["usage"], axis=1)
        .reset_index(drop=True)
    )
    test_data = (
        encorded_data[encorded_data["usage"] == 1]
        .drop(["usage"], axis=1)
        .reset_index(drop=True)
    )

    train_y = train_data.loc[:, "y"]
    train_x = train_data.drop(["y"], axis=1)
    test_x = test_data.drop(["y"], axis=1)

    ids = test_x.loc[:, "id"]

    # # 学習用のハイパラをチューニング
    # best_params = get_best_params(train_x, train_y)

    # 学習
    n_splits = 5
    kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=0)
    models = train(train_x, train_y, kfold)

    # 評価
    threshold = 0.5
    y_preds = []
    for i, model in enumerate(models):
        y_pred = predict(model, test_x, threshold)
        y_preds.append(y_pred)

    # 提出用ファイル成型
    winner_pred = pd.concat(y_preds, axis=1).mode(axis=1).rename(columns={0: "y"})
    submission = pd.concat([ids, winner_pred], axis=1)

    print(submission)
    submission.to_csv(f"{DATA_DIR}/submission18.csv", index=False)

if __name__ == "__main__":
    run_all()


import math
import random
import os
from datetime import datetime

from tornado.web import RequestHandler

from wx.basic import Basic
from common.redis_conn import redis

class UploadHandler(RequestHandler):
    def get(self):
        accessToken = Basic().get_access_token()
        return self.render('upload.html', accessToken=accessToken)

    def post(self):
        files = self.request.files.get('img', None)
        if files:
            if not files:
                return self.write('上传图片不能为空')

            upload_path = os.path.join(os.path.dirname(__file__), 'upload')
            if not os.path.exists(upload_path):
                os.mkdir(upload_path)

            for file in files:
                if file['content_type'] not in ['image/png', 'image/jpeg']:
                    return self.write('上传图片格式有误')

                try:
                    name = datetime.strftime(datetime.now(), '%Y%m%d%H%M%S') + str(math.floor(random.random() * 10))
                    sName = name + '.' + file['filename'].split('.')[1]
                    with open(os.path.join(upload_path, sName), 'wb') as up:
                        up.write(file['body'])

                    accessToken = Basic().get_access_token()
                    cmd = 'curl -F media=@upload/%s "https://api.weixin.qq.com/cgi-bin/media/uploadimg?access_token=%s"' % (
                        sName, accessToken)
                    wxName = json.loads(os.popen(cmd).read())['url']
                    fName = file['filename']

                    redis.hmset('wx:' + name, {'sname': sName, 'wxname': wxName, 'fname': fName})
                except Exception:
                    print(file['filename'] + '文件上传失败')

            return self.write('图片上传成功')


import astropy.units as u
import gala.dynamics as gd
import gala.integrate as gi
import numpy as np
from scipy.optimize import minimize

from .actions_o2gf import get_o2gf_aaf
from .actions_staeckel import get_staeckel_aaf
from .config import vcirc
from .potentials import galpy_potentials, potentials

def _same_actions_objfunc_staeckel(p, pos, vy, potential_name, match_actions):
    vx, vz = p
    w0 = gd.PhaseSpacePosition(pos=pos, vel=[vx, vy, vz] * u.km / u.s)

    o = potentials[potential_name].integrate_orbit(
        w0, dt=1.0 * u.Myr, t1=0, t2=1 * u.Gyr, Integrator=gi.DOPRI853Integrator
    )
    aaf = get_staeckel_aaf(o[::2], galpy_potentials[potential_name])
    actions = aaf["actions"].mean(axis=-1)

    _unit = (u.km / u.s * u.kpc) ** 2
    val = (
        (actions[0] - match_actions[0]) ** 2 + (actions[2] - match_actions[2]) ** 2
    ).to_value(_unit)

    return val

def _same_actions_objfunc_sanders(p, pos, vy, potential_name, match_actions):
    vx, vz = p
    w0 = gd.PhaseSpacePosition(pos=pos, vel=[vx, vy, vz] * u.km / u.s)
    aaf = get_o2gf_aaf(potentials[potential_name], w0, N_max=8)
    actions = aaf["actions"]

    _unit = (u.km / u.s * u.kpc) ** 2
    val = (
        (actions[0] - match_actions[0]) ** 2 + (actions[2] - match_actions[2]) ** 2
    ).to_value(_unit)

    return val

def get_w0s_with_same_actions(fiducial_w0, vy=None, staeckel=False):
    if staeckel:
        _same_actions_objfunc = _same_actions_objfunc_staeckel
    else:
        _same_actions_objfunc = _same_actions_objfunc_sanders

    if vy is None:
        # Default: set to circular velocity
        vy = vcirc

    # First, determine actions for the input orbit in the
    # fiducial potential model. These will be the target action values
    fiducial_actions = []
    for n in range(fiducial_w0.shape[0]):
        if staeckel:
            o = potentials["1.0"].integrate_orbit(
                fiducial_w0[n], dt=0.5, t1=0, t2=2 * u.Gyr
            )  # MAGIC NUMBERS
            fiducial_actions.append(
                get_staeckel_aaf(o, galpy_potentials["1.0"])["actions"]
            )
        else:
            fiducial_actions.append(
                get_o2gf_aaf(potentials["1.0"], fiducial_w0[n], N_max=8)["actions"]
            )

    fiducial_actions = u.Quantity(fiducial_actions).to(u.km / u.s * u.kpc)

    if staeckel:
        fiducial_actions = np.mean(fiducial_actions, axis=-1)

    w0s = {}
    for name in potentials:
        if name == "1.0":
            w0s[name] = fiducial_w0
            continue

        w0s[name] = []
        for n in range(fiducial_w0.shape[0]):
            res = minimize(
                _same_actions_objfunc,
                x0=fiducial_w0.v_xyz.value[[0, 2], n],
                args=(
                    fiducial_w0.pos[n],
                    vy.to_value(u.km / u.s),
                    name,
                    fiducial_actions[n],
                ),
                method="nelder-mead",
                options=dict(maxfev=64),
            )

            if res.fun > 1e-3:
                print(f"{name}, {n}: func val = {res.fun} -- " "Failed to converge")

            w0s[name].append(
                gd.PhaseSpacePosition(
                    pos=fiducial_w0.pos[n],
                    vel=[res.x[0], vy.to_value(u.km / u.s), res.x[1]] * u.km / u.s,
                )
            )

        w0s[name] = gd.combine(w0s[name])

    return w0s