from typing import Optional
import time
import logging

from fastapi import APIRouter, Form, Depends

from service.grants import (
    ServiceFactory,
    GrantsService,
    authorization_code_service,
    client_credentials_service,
    refresh_token_service,
    resource_owner_service,
    refresh_token_service,
)

from service.client_service import ClientService
from service.user_service import UserService

from models.auth import AuthModel, get_auth_model
from models.client import ClientModel, get_client_model

from entity.oauth import Code

from exception import (
    UnauthorizedClientException,
    InvalidTokenException,
    TokenExpiredException,
    InvalidRequestException
)

from util.jwt import MyJWT

logger = logging.getLogger("uvicorn")

router = APIRouter()

@router.post("/v1/code", tags=["token"])
async def code(
    client_id: str = Form(...),
    scope: Optional[str] = Form(None),
    nonce: Optional[str] = Form(None),
    code_challenge: Optional[str] = Form(None),
    code_challenge_method: Optional[str] = Form(None),
    user: UserService = Depends(UserService),
    auth_model: AuthModel = Depends(get_auth_model),
    client_model: ClientModel = Depends(get_client_model),
):
    client = client_model.readClient(client_id)
    if not client:
        raise UnauthorizedClientException()

    user.verify()

    code = Code(**{
        "client_id": client_id,
        "username": user.username,
        "scope": scope,
        "nonce": nonce,
        "code_challenge": code_challenge,
        "code_challenge_method": code_challenge_method,
    })

    auth_model.saveCode(code)

    return {
        "code": code.key,
        "expire_in": code.expire_in,
    }

@router.post("/v1/id_token", tags=["token"])
async def id_token(
    client_id: str = Form(...),
    nonce: Optional[str] = Form(None),
    user: UserService = Depends(UserService),
    client_model: ClientModel = Depends(get_client_model),
):
    client = client_model.readClient(client_id)
    if not client:
        raise UnauthorizedClientException()

    user.verify()

    payload = {
        "iss": "http://localhost:8080",
        "sub": user.username,
        "aud": client_id,
        "exp": int(time.time()) + 3600,
        "iat": int(time.time()),
        "auth_time": int(time.time()),
        "nonce": nonce,
    }

    return {
        "id_token": MyJWT.encode(payload, "secret", algorithm="HS256"),
    }

@router.post("/v1/token", tags=["token"], dependencies=[
    Depends(authorization_code_service.validation),
    Depends(resource_owner_service.validation),
    Depends(refresh_token_service.validation),
    Depends(client_credentials_service.validation),
])
async def token(
    logic: GrantsService = Depends(ServiceFactory)
):
    logic.verify()

    return logic.generate_token()

@router.post("/v1/token/info", tags=["token"])
async def token_info(
    token: str = Form(...),
    client: ClientService = Depends(ClientService),
    model: AuthModel = Depends(get_auth_model),
):
    client.verify()

    access_token = model.readToken(token)
    if not access_token:
        raise InvalidTokenException()

    if access_token.isExpired():
        raise TokenExpiredException()

    if not access_token.equals(client.client_id):
        raise InvalidRequestException()

    return {
        "username": access_token.username,
        "scope": access_token.scope,
        "expire_in": access_token.expire_in,
    }


#!/usr/bin/env python
# -*- coding: utf8 -*-
import numpy as np
import matplotlib.pyplot as plt
from sklearn import linear_model

def plot(matrix, labels, ntrain=None):
    nsamples, nfeatures = matrix.shape
    fig, (col1, col2) = plt.subplots(1, 2, sharey=True)
    col1.autoscale(False)
    col2.autoscale(False)
    col1.set_xlim((0, nfeatures))
    col1.set_ylim((0, nsamples))
    matrix_normed = matrix / np.max(matrix, axis=1)[:, None]
    col1.imshow(matrix_normed, origin='lower', cmap='viridis', aspect='auto')
    col2.plot(labels[:, 8] / labels[:, 8].max(), range(nsamples))
    col2.plot(labels[:, 10] / labels[:, 10].max(), range(nsamples))
    if ntrain is not None:
        col1.axhline(ntrain, color='white', linewidth=2, alpha=0.5)
        col2.axhline(ntrain, color='red')
    col1.set_xlabel('features')
    col1.set_ylabel('examples')
    col2.set_xlabel('label')

def main():
    dataset = np.load('dataset.npz')
    design_matrix = dataset['source_time_functions']
    # Decimate the design_matrix, using one feature every ten
    design_matrix = design_matrix[:, ::10]
    nsamples, nfeatures = design_matrix.shape
    print(nsamples, nfeatures)
    labels = dataset['labels']

    # Number of examples to use for training
    # ntrain = 1000
    ntrain = 500

    # label indices:
    # 0: year, 1: month, 2:day, 3:hour, 4:min, 5:sec, 6:lat, 7:lon
    # 8: depth, 9: M0, 10: Mw, 11:strike1, 12: dip1, 13: rake1, 14:strike2,
    # 15:dip2, 16:rake2
    plot(design_matrix, labels, ntrain)

    # normalize each row of the design matrix
    design_matrix /= design_matrix.max(axis=1)[:, None]

    all_mag = labels[:, 10]

    training_matrix = design_matrix[0:ntrain, :]
    training_mag = all_mag[0:ntrain]

    reg = linear_model.LinearRegression()
    reg.fit(training_matrix, training_mag)

    predicted_mag = reg.predict(design_matrix)
    fig, ax = plt.subplots()
    ax.set_xlim((0, nsamples))
    ax.plot(all_mag-predicted_mag)
    ax.axvline(ntrain, color='red')
    ax.set_xlabel('example')
    ax.set_ylabel('delta mag')

    fig, ax = plt.subplots()
    ax.plot(reg.coef_)
    ax.set_xlabel('coefficient number')
    ax.set_ylabel('coefficient value')

    plt.show()

if __name__ == "__main__":
    main()


from ipykernel.kernelbase import Kernel
from contextlib import contextmanager
import os
import sys
import tempfile
import platform
import ctypes

OS = platform.platform()
libc = ctypes.CDLL(None)
if 'Darwin' in OS:
    c_stdout = ctypes.c_void_p.in_dll(libc, '__stdoutp')
elif 'macOS' in OS:
    c_stdout = ctypes.c_void_p.in_dll(libc, '__stdoutp')
elif 'Linux' in OS:
    c_stdout = ctypes.c_void_p.in_dll(libc, 'stdout')

else:
    print("""Your OS is not currently compatible with this Kernel \n
             Please make an issue here: \n
             https://github.com/ryanpepper/tcl_kernel
             """)
try:
    import io
    import Tkinter
except ImportError:
    import tkinter as Tkinter
__version__ = '0.0.4'

@contextmanager
def stdout_redirector(stream):
    # The original fd stdout points to. Usually 1 on POSIX systems.
    original_stdout_fd = sys.__stdout__.fileno()

    def _redirect_stdout(to_fd):
        """Redirect stdout to the given file descriptor."""
        # Flush the C-level buffer stdout
        libc.fflush(c_stdout)
        # Flush and close sys.stdout - also closes the file descriptor (fd)
        sys.stdout.close()
        # Make original_stdout_fd point to the same file as to_fd
        os.dup2(to_fd, original_stdout_fd)
        # Create a new sys.stdout that points to the redirected fd
        sys.stdout = io.TextIOWrapper(os.fdopen(original_stdout_fd, 'wb'))

    # Save a copy of the original stdout fd in saved_stdout_fd
    saved_stdout_fd = os.dup(original_stdout_fd)
    try:
        # Create a temporary file and redirect stdout to it
        tfile = tempfile.TemporaryFile(mode='w+b')
        _redirect_stdout(tfile.fileno())
        # Yield to caller, then redirect stdout back to the saved fd
        yield
        _redirect_stdout(saved_stdout_fd)
        # Copy contents of temporary file to the given stream
        tfile.flush()
        tfile.seek(0, io.SEEK_SET)
        stream.write(tfile.read())
    finally:
        tfile.close()
        os.close(saved_stdout_fd)

class TclKernel(Kernel):

    implementation = 'tcl_kernel'
    implementation_version = __version__
    language_info = {"name": "Tcl",
                     "codemirror_mode": {
                         "version": 2,
                         "name": "text/x-tcl"
                     },
                     "mimetype": "text/x-tcl",
                     "file_extension": ".tcl"}

    _banner = "Tcl Kernel"

    @property
    def banner(self):
        if self._banner is None:
            banner_undecoded = self._run_code('puts [info patchlevel]')
            self._banner = banner_undecoded.decode('utf-8')
        return self._banner

    def __init__(self, **kwargs):
        Kernel.__init__(self, **kwargs)
        self.tcl = Tkinter.Tcl()
        self.execution_count = 0

    def _run_code(self, code, decodestr='utf-8'):
        temp_stdout_stream = io.BytesIO()
        stderr_output = None
        returnval = ''
        with stdout_redirector(temp_stdout_stream):
            try:
                returnval = self.tcl.eval(code)
            except Tkinter.TclError as scripterr:
                stderr_output = scripterr.args[0]
        output = temp_stdout_stream.getvalue()
        temp_stdout_stream.close()
        return (output.decode(decodestr) + returnval,
                stderr_output)

    def do_execute(self, code, silent, store_history=True,
                   user_expressions=None, allow_stdin=False):
        output, stderr_output = self._run_code(code)

        if not silent:
            stream_content = {
                'name': 'stdout', 'text': output}
            self.send_response(self.iopub_socket, 'stream', stream_content)
            if stderr_output:
                stream_content = {
                    'name': 'stderr', 'text': stderr_output}
                self.send_response(self.iopub_socket, 'stream', stream_content)

        return {'status': 'ok', 'execution_count': self.execution_count,
                'payload': [], 'user_expressions': {}}


import discord
from discord.ext import commands
import random

class Toys():
    def __init__(self, bot):
        self.bot = bot

    @commands.command()
    async def coin(self, flips=1):
        """
        Flip a coin! Defaults to one flip, add number to flip multiple times, with
        a maximum of 5 flips.

        Syntax: !coin, !coin 5
        """
        if flips > 5:
            await self.bot.say('Input too large.')
        else:
            for i in range(0,flips):
                flip = random.randint(1,2)
                if flip == 1:
                    await self.bot.say('<:tracer:311966007746887683>')
                else:
                    await self.bot.say('<:widow:311966007935762432>')

def setup(bot):
    bot.add_cog(Toys(bot))


from flask import Blueprint, request, jsonify, Response
from database.models import City, Sqli, User, Xss
from database.db import db
from flask_cors import CORS, cross_origin
from database.schemas import CitySchema
from flask_restful import Resource
import json
import sys
from marshmallow import ValidationError
from flask_jwt_extended import get_jwt_identity, jwt_required
from ml_models.sqli_model import predict_sqli
from ml_models.xss_model import predict_xss

city_schema = CitySchema()
cities_schema = CitySchema(many=True)

class CitiesApi(Resource):
    @jwt_required
    @cross_origin()
    def get(self):
        all_cities = City.query.all()
        print(City.query.count())
        result = cities_schema.dump(all_cities)
        return jsonify({'data': result})

    @jwt_required
    @cross_origin()
    def post(self):
        try:
            city_schema.load(request.json)
        except ValidationError as err:
            print(err.messages)
            return jsonify(err.messages), 500
        name = request.json['name']
        country_id = request.json['countryId']
        username = get_jwt_identity()
        user_id = User.query.filter(User.username == username).first().id
        value = predict_sqli(name)
        xss_value = predict_xss(name)
        new_xss = Xss(xss_value, user_id, False, name)
        new_product = City(name, country_id)
        new_sqli = Sqli(value, user_id, False, name)
        db.session.add(new_xss)
        db.session.add(new_sqli)
        db.session.add(new_product)
        value = predict_sqli(str(country_id))
        new_sqli = Sqli(value, user_id, False, str(country_id))
        db.session.add(new_sqli)
        db.session.commit()
        return city_schema.jsonify({'data': new_product})

    @jwt_required
    @cross_origin()
    def options(self):
        return jsonify()

class CityApi(Resource):
    @jwt_required
    @cross_origin()
    def get(self, id):
        city = City.query.get(id)
        return city_schema.jsonify(city)

    @jwt_required
    @cross_origin()
    def put(self, id):
        try:
            city_schema.load(request.json)
        except ValidationError as err:
            return jsonify(err.messages), 500
        city = City.query.get(id)
        name = request.json['name']
        country_id = request.json['countryId']
        city.name = name
        city.country_id = country_id
        city.updated_on = db.func.now()
        username = get_jwt_identity()
        user_id = User.query.filter(User.username == username).first().id
        value = predict_sqli(name)
        xss_value = predict_xss(name)
        new_sqli = Sqli(value, user_id, False, name)
        new_xss = Xss(xss_value, user_id, False, name)
        db.session.add(new_sqli)
        db.session.add(new_xss)
        value = predict_sqli(str(country_id))
        new_sqli = Sqli(value, user_id, False, str(country_id))
        db.session.commit()
        return city_schema.jsonify({'data': city})

    @jwt_required
    @cross_origin()
    def delete(self, id):
        city = City.query.get(id)
        db.session.delete(city)
        db.session.commit()
        return city_schema.jsonify(city)

    @jwt_required
    @cross_origin()
    def options(self):
        return jsonify()


import logging
import os
from multiprocessing import Pool
from uuid import uuid4

import numpy as np

#
from ..common_vars.directories import LUIGI_OUT_FOLDER, POSTPROCESS_OUT_FOLDER
from ..common_vars.regions import face4id
from ..common_vars.time_slices import idx_t
from ..isotropic_spectra.co_spec import cospec_ab
from ..isotropic_spectra.coherence import coherence_ab
# Spectral analysis
from ..isotropic_spectra.isotropic import calc_ispec
from .output import VorticityGrid

## PATHS
ds_path_fmt = LUIGI_OUT_FOLDER+"/Datasets_compressed/{}/{}"
dxdy_fname_fmt = LUIGI_OUT_FOLDER+"/Datasets_compressed/{}/{}/{}.txt"
uv_fname_fmt = LUIGI_OUT_FOLDER+"/Datasets_compressed/{0}/{1}/{2}{3:02d}_{4:05d}.npz"
spectra_folder= POSTPROCESS_OUT_FOLDER+"/wk_spectra"
spectra_fn_fmt = "{folder}/{id}{tag}_{t_res}.npz"

##
def _var4IdTs(regionId, var_name, t_idx_model, Z_idx, timeRes, threadId):
  #print(var_name,t_idx,)
  fname_hf = uv_fname_fmt.format(regionId, timeRes, var_name, Z_idx, t_idx_model)
  try:
    return np.load(fname_hf)["uv"]
  except Exception as err:
    logging.warn("NP load (file: {}) error: {}".format(fname_hf, err))
    return np.load(fname_hf, allow_pickle=True)["uv"]

## Aux function to load vars in parallel (duh!)
## ThreadId is only here to avoid function signature collision
def _varLoader(xyshape, regionId, var_name, t_slice, Z_idx, timeRes, threadId):
  # Create empty matrix
  shape = (xyshape[0], xyshape[1], len(t_slice))
  V = np.zeros(shape)
  logging.debug("Loading {} (thread: {}): shape (k={}): {}".format(var_name, threadId, Z_idx, shape))

  for idx,t in enumerate(t_slice):
    V_ = _var4IdTs(regionId, var_name, t, Z_idx, timeRes, threadId)
    V[:,:,idx] = V_

  return V

class LLCRegion():
  __vars = None
  __grid = None
  __regionId = None
  __timeRes = None
  __tag = None
  __timeVec = None
  __face = None
  __threads = 4
  __dt = None
  __dxAvg = None
  __dyAvg = None
  __spectra = None

  def __init__(self, rid, timeVec, tag=None, t_res="hours", threads=4):
    self.__vars = {}
    self.__regionId = rid
    self.__timeRes = t_res
    self.__timeVec = timeVec
    self.__tag = "" if tag is None else "_{}".format(tag)
    self.__face = face4id[rid]
    self.__grid = VorticityGrid(rid, t_res)
    self.__threads = threads
    self.__dt = 1 if t_res=="hours" else 24
    self.__dxAvg = np.mean(self.__grid.dxg)/1000
    self.__dyAvg = np.mean(self.__grid.dyg)/1000
    logging.info("Grid: dx = {} km, dy = {} km, dt = {} h".format(self.__dxAvg, self.__dyAvg, self.__dt))
    self.__spectra = dict(self.load_spectra()) if self.spectra_exists() else dict({})
    spec_vars = [k for k in self.__spectra.keys()]
    logging.info("Spectra - Variables: {}".format(spec_vars))

  def getGrid(self):
    return self.__grid

  def getGridC(self):
    return self.__grid.xyc

  def getGridG(self):
    return self.__grid.xyg

  def varForId(self, var_name, Z_idx=0, t_firstaxis=False):
    # Create empty matrix
    shape_uv = self.__grid.f.shape
    shape = (shape_uv[0], shape_uv[1], len(self.__timeVec))
    logging.info("Loading {}: shape (k={}): {}".format(var_name, Z_idx, shape))
    V = np.zeros(shape)

    nLoaders = self.__threads
    with Pool(nLoaders) as pool:
      args = [(shape_uv, self.__regionId, var_name, self.__timeVec[ii::nLoaders], Z_idx, self.__timeRes, uuid4()) for ii in range(nLoaders)]
      res = [pool.apply_async(_varLoader, a) for a in args]
      for ii,r in enumerate(res):
        logging.debug("Geting result {} - {}/{}".format(var_name, ii+1, nLoaders))
        V[:,:,ii::nLoaders] = r.get()
        logging.debug("Got result {} - {}/{}".format(var_name, ii+1, nLoaders))

    if t_firstaxis:
      V = np.moveaxis(V, -1, 0)

    return V

  def loadScalar(self, var_name, Z_idx=0, t_firstaxis=False):
    self.__vars[var_name] = self.varForId(var_name, Z_idx=Z_idx, t_firstaxis=t_firstaxis)

  def loadHorizontalVector(self, x_var_name, y_var_name, out_var_name, Z_idx=0, t_firstaxis=False):
    xVec = self.varForId(x_var_name, Z_idx=Z_idx, t_firstaxis=t_firstaxis)
    yVec = self.varForId(y_var_name, Z_idx=Z_idx, t_firstaxis=t_firstaxis)

    # Para face>6, los vectores (U,V) están en las coordenadas "locales"
    # Ver: https://github.com/MITgcm/MITgcm/issues/248 and https://github.com/MITgcm/xmitgcm/issues/204

    if self.__face>6:
       xVec,yVec = yVec,-1*xVec

    self.__vars[out_var_name] = (xVec, yVec)

  def get(self, var_name):
    return self.__vars[var_name]

  def norm(self, vec_var_name, out_var_name):
    logging.info("Calculating {} = |{}|".format(out_var_name, vec_var_name))
    xVec,yVec = self.get(vec_var_name)
    self.__vars[out_var_name] = np.sqrt(xVec**2 + yVec**2)

  def divergence(self, vec_var_name, out_var_name):
    logging.info("Calculating {} = div({})".format(out_var_name, vec_var_name))
    xVec,yVec = self.get(vec_var_name)
    div = self.__grid.div(np.moveaxis(xVec, -1, 0), np.moveaxis(yVec, -1, 0))
    self.__vars[out_var_name] = np.moveaxis(div, 0, -1)

  def hcurl(self, vec_var_name, out_var_name):
    logging.info("Calculating {} = curl_h({})".format(out_var_name, vec_var_name))
    xVec,yVec = self.get(vec_var_name)
    rot = self.__grid.rv(np.moveaxis(xVec, -1, 0), np.moveaxis(yVec, -1, 0))
    self.__vars[out_var_name] = np.moveaxis(rot, 0, -1)

  ##Spectral analysis
  def get_spectrum(self, spectrum_name):
    return self.__spectra[spectrum_name]

  def get_spectra_names(self):
    return self.__spectra.keys()

  def _cospectrum(self, A, B):
    return cospec_ab(A, B, self.__dxAvg, self.__dyAvg, self.__dt)

  def _coh(self, A, B):
    return coherence_ab(A, B, self.__dxAvg, self.__dyAvg, self.__dt)

  def power_spectrum_1d(self, var_name, spectrum_name, recalculate=False):
    if spectrum_name in self.__spectra.keys() and not recalculate:
      logging.info("Spectrum {} already there".format(spectrum_name))
    else:
      logging.info("Calculating {} = FFT_pow({})".format(spectrum_name, var_name))
      hasVar = var_name in self.__vars.keys()
      V = self.__vars[var_name] if hasVar else self.loadScalar(var_name)
      powSpec, kx, ky, omega, dkx, dky, domega = self._cospectrum(V, V)
      kiso, powSpec_iso = calc_ispec(powSpec, kx, ky, omega)
      self.__spectra["k_h"] = kiso
      self.__spectra["om"] = omega
      self.__spectra[spectrum_name] = powSpec_iso
      logging.info("Saved {}({}). min: {}, max: {}".format(spectrum_name, self.__spectra[spectrum_name].shape, np.min(self.__spectra[spectrum_name]), np.max(self.__spectra[spectrum_name])))

  def power_spectrum_2d(self, var_name, spectrum_name, recalculate=False):
    if spectrum_name in self.__spectra.keys() and not recalculate:
      logging.info("Spectrum {} already there".format(spectrum_name))
    else:
      logging.info("Calculating {0} = FFT_pow({1}_x) + FFT_pow({1}_y)".format(spectrum_name, var_name))
      hasVec = var_name in self.__vars.keys()
      if not hasVec:
        logging.warn("First load vector into {} -- Vars: {}".format(var_name, self.__vars.keys()))
        return
      Vx, Vy = self.__vars[var_name]
      powSpecX, kx, ky, omega, dkx, dky, domega = self._cospectrum(Vx, Vx)
      powSpecY, kx, ky, omega, dkx, dky, domega = self._cospectrum(Vy, Vy)
      kiso, powSpecX_iso = calc_ispec(powSpecX, kx, ky, omega)
      kiso, powSpecY_iso = calc_ispec(powSpecX, kx, ky, omega)
      self.__spectra["k_h"] = kiso
      self.__spectra["om"] = omega
      self.__spectra["{}_x".format(spectrum_name)] = powSpecX_iso
      self.__spectra["{}_y".format(spectrum_name)] = powSpecY_iso
      self.__spectra[spectrum_name] = powSpecX_iso + powSpecY_iso
      logging.info("Saved {}({}). min: {}, max: {}".format(spectrum_name, self.__spectra[spectrum_name].shape, np.min(self.__spectra[spectrum_name]), np.max(self.__spectra[spectrum_name])))

  def cospectrum(self, var1_name, var2_name, spectrum_name, recalculate=False):
    if spectrum_name in self.__spectra.keys() and not recalculate:
      logging.info("Cospectrum {} already there".format(spectrum_name))
    else:
      logging.info("Calculating {} = FFT({})*FFT({})^*".format(spectrum_name, var1_name, var2_name))
      hasVar1 = var1_name in self.__vars.keys()
      hasVar2 = var2_name in self.__vars.keys()
      if not (hasVar1 and hasVar2):
        logging.warn("First load vars ({}, {}) -- Vars: {}".format(var1_name, var2_name, self.__vars.keys()))
        return
      A, B = self.__vars[var1_name], self.__vars[var2_name]
      cospec, kx, ky, omega, dkx, dky, domega = self._cospectrum(A, B)
      kiso, cospec_iso = calc_ispec(cospec, kx, ky, omega)
      self.__spectra["k_h"] = kiso
      self.__spectra["om"] = omega
      self.__spectra[spectrum_name] = cospec_iso
      logging.info("Saved {}({}). min: {}, max: {}".format(spectrum_name, self.__spectra[spectrum_name].shape, np.min(self.__spectra[spectrum_name]), np.max(self.__spectra[spectrum_name])))

  def coherence(self, var1_name, var2_name, spectrum_name, recalculate=False):
    if spectrum_name in self.__spectra.keys() and not recalculate:
      logging.info("Coherence {} already there".format(spectrum_name))
    else:
      logging.info("Calculating {} = FFT({})*FFT({})^*".format(spectrum_name, var1_name, var2_name))
      hasVar1 = var1_name in self.__vars.keys()
      hasVar2 = var2_name in self.__vars.keys()
      if not (hasVar1 and hasVar2):
        logging.warn("Please first load vars ({}, {}) -- Loaded vars: {}".format(var1_name, var2_name, self.__vars.keys()))
        return
      A, B = self.__vars[var1_name], self.__vars[var2_name]
      logging.info("Calculating coh({},{})".format(var1_name,var2_name))
      coh, kx, ky, omega, dkx, dky, domega = self._coh(A, B)
      kiso, coh_iso = calc_ispec(coh, kx, ky, omega)
      self.__spectra["k_h"] = kiso
      self.__spectra["om"] = omega
      self.__spectra[spectrum_name] = coh_iso
      logging.info("Saved {}({}). min: {}, max: {}".format(spectrum_name, self.__spectra[spectrum_name].shape, np.min(self.__spectra[spectrum_name]), np.max(self.__spectra[spectrum_name])))

  def coherence_error(self, spectrum_name, nd=1):
    logging.info("Calculating normalized error E[{}]".format(spectrum_name))
    coh_spec = self.__spectra[spectrum_name]
    abs_coh = np.abs(coh_spec)
    sqr_coh = coh_spec**2
    self.__spectra["{}_norm_err".format(spectrum_name)] = np.sqrt(2/nd)*(1-sqr_coh)/abs_coh
    logging.info("Saved E[{}]".format(spectrum_name))

  def coh_err_mask(self, spectrum_name, nd=1):
    logging.info("Mask for values where 1>=2*E[{}]".format(spectrum_name))
    norm_err = self.__spectra["{}_norm_err".format(spectrum_name)]
    self.__spectra["{}_err_mask".format(spectrum_name)] = (2*norm_err<=1)
    logging.info("Saved {}_err_mask".format(spectrum_name))

  def spectra_exists(self):
    spectra_fn = spectra_fn_fmt.format(folder=spectra_folder, id=self.__regionId, tag=self.__tag, t_res=self.__timeRes)
    logging.info("Exists? {}: {} -- Filename {}".format(self.__regionId, self.__tag, spectra_fn))
    return os.path.exists(spectra_fn)

  def save_spectra(self):
    spectra_fn = spectra_fn_fmt.format(folder=spectra_folder, id=self.__regionId, tag=self.__tag, t_res=self.__timeRes)
    logging.debug("{}: {} -- Filename {}".format(self.__regionId, self.__tag, spectra_fn))
    self.ensure_dir(spectra_fn)
    np.savez_compressed(spectra_fn, **self.__spectra)
    spec_vars = [k for k in self.__spectra.keys()]
    logging.info("Saved spectra {}: {}".format(spectra_fn, spec_vars))

  def load_spectra(self):
    spectra_fn = spectra_fn_fmt.format(folder=spectra_folder, id=self.__regionId, tag=self.__tag, t_res=self.__timeRes)
    logging.debug("{}: {} -- Opening {}".format(self.__regionId, self.__tag, spectra_fn))
    return np.load(spectra_fn)

## Tools
  def ensure_dir(self, file_path):
    directory = os.path.dirname(file_path)
    if not os.path.exists(directory):
      os.makedirs(directory)


import json

import joblib
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.lines import Line2D
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.feature_selection import SelectKBest, chi2
from sklearn.metrics import make_scorer, accuracy_score, precision_score, recall_score, f1_score, classification_report, \
    confusion_matrix, ConfusionMatrixDisplay
from sklearn.model_selection import train_test_split, GridSearchCV

from sklearn.naive_bayes import ComplementNB
from sklearn.pipeline import Pipeline
from sklearn.linear_model import LogisticRegression
from sklearn import svm

import utils as u

#tweet fields
# id
# date
# content
# likeCount
# lang
from pre_processing import clening
from text_normalization import normalize_text

def filter_by_lang(tweets):
    return tweets[tweets['lang'] == 'it']

def date_format(tweets):
    return pd.to_datetime(tweets["date"]).dt.strftime("%m-%d-%Y")

def generate_training_sets(pick):

    old_set = pd.read_csv('../dataset/old_training_set_0.csv')
    old_set = old_set[~old_set['sentiment'].isnull()]
    #print(old_training_set.shape)

    new_labeled_tweets = pd.read_csv('../dataset/pick_' + str(pick) + '.csv')
    new_labeled_tweets = new_labeled_tweets[~new_labeled_tweets['sentiment'].isnull()]
    new_labeled_tweets = new_labeled_tweets[['id', 'content', 'date', 'sentiment']]
    #print(new_labeled_tweets.shape)

    # incremental model
    incremental_set = pd.concat([old_set, new_labeled_tweets], axis=0)
    #print(incremental_training.shape)
    u.save_dataset(incremental_set, 'incremental_pick_' + str(pick))

    # Sliding model
    old_set.drop(old_set.head(360).index, inplace=True)
    sliding_set = pd.concat([old_set, new_labeled_tweets], axis=0)
    #print(sliding_training.shape)
    u.save_dataset(sliding_set, 'sliding_pick_' + str(pick))
    return incremental_set, sliding_set

complementNB = Pipeline([
    ('vect', CountVectorizer(ngram_range=(1, 1), max_df=1.0)),
    ('tfidf', TfidfTransformer()),
    ('clf', ComplementNB()),
])

logreg = Pipeline([
    ('vect', CountVectorizer(ngram_range=(1,1), max_df=1.0)),
    ('tfidf', TfidfTransformer()),
    ('clf', LogisticRegression()),
])

svm = Pipeline([
    ('vect', CountVectorizer(ngram_range=(1,1), max_df=1.0)),
    ('tfidf', TfidfTransformer()),
    ('clf', svm.SVC()),
])

pipelines = [
    {'name': 'ComplementNB', 'pipeline': complementNB},
    {'name': 'LogisticRegression', 'pipeline': logreg},
    {'name': 'SVM', 'pipeline': svm}
]

def train_classifier(clf, name, mode, peak, x_train, y_train):

    model = clf.fit(x_train, y_train)

    # saving best estimator
    model_name = './models/'+name+'_'+mode+'_model_peak_' + str(peak) + ".pkl"
    joblib.dump(model, model_name, compress=1)

    print(len(clf[0].get_feature_names()))

def evaluate_classifier(clf, name, mode, peak, x_test, y_test):

    # Evaluation on test set
    predicted = clf.predict(x_test)  # prediction
    print(clf.classes_, 'Negative', 'Neutral', 'Positive')
    # Extracting statistics and metrics
    report = classification_report(y_test, predicted, labels=clf.classes_, target_names=['Negative', 'Neutral', 'Positive'], digits=4, output_dict=True)
    print(report)

    clsf_report = pd.DataFrame(report).transpose()
    clsf_report.to_csv('./Monitoring_Results/'+name+'_'+mode+'_test_result_peak_'+str(peak)+'.csv', index=True)

def generate_models(peak, pipe):

    incremental_set = pd.read_csv('../dataset/incremental_peak_' + str(peak) + '.csv')
    sliding_set = pd.read_csv('../dataset/sliding_peak_' + str(peak) + '.csv')

    incremental_set = clening(incremental_set)
    sliding_set = clening(sliding_set)

    incremental_data = normalize_text(incremental_set['content'])
    incremental_labels = incremental_set['sentiment']

    sliding_data = normalize_text(sliding_set['content'])
    sliding_labels = sliding_set['sentiment']

    train_classifier(pipe['pipeline'], pipe['name'], 'incremental', peak, incremental_data, incremental_labels)
    train_classifier(pipe['pipeline'], pipe['name'], 'sliding', peak, sliding_data, sliding_labels)

    incremental_model = joblib.load('models/' + pipe['name'] + '_incremental_model_peak_' + str(peak) + '.pkl')
    sliding_model = joblib.load('models/' + pipe['name'] + '_sliding_model_peak_' + str(peak) + '.pkl')
    static_model = joblib.load('models/' + pipe['name'] + '.pkl')

    # test the models on newest tweets
    next_peak = pd.read_csv('../dataset/peak_'+str(peak+1)+'.csv')
    next_peak = next_peak[~next_peak['sentiment'].isnull()]
    print(next_peak.shape)
    next_peak = clening(next_peak)
    next_peak_data = normalize_text(next_peak['content'])
    next_peak_label = next_peak['sentiment']

    evaluate_classifier(incremental_model, pipe['name'], 'incremental', peak, next_peak_data, next_peak_label)
    evaluate_classifier(sliding_model, pipe['name'], 'sliding', peak, next_peak_data, next_peak_label)
    evaluate_classifier(static_model, pipe['name'], 'static', peak, next_peak_data, next_peak_label)

if __name__ == '__main__':
    for pipe in pipelines:
        generate_models(6, pipe)

    pass


"""Database model classes"""
import datetime
import json

from bot.utils import constants

DEFAULT_SETTINGS = {"settings": {"prefix": constants.Bot.def_prefix}}

class Guild:
    """Guild information from Kat API."""

    __slots__ = "guild_id", "_settings"

    def __init__(self, id, _settings: dict):
        self.guild_id = id
        self._settings: dict = _settings

    @classmethod
    def from_dict(cls, data: dict):
        return cls(data["id"], dict(data["settings"]))

    @classmethod
    async def get(cls, id, session):
        data = await session.get("guilds/" + str(id))
        if data[0]:
            return cls.from_dict(data[0])
        else:
            return cls.from_dict({"id": id, "settings": DEFAULT_SETTINGS})

    @classmethod
    async def members(cls, id, session):
        data = await session.get(f"guilds/{id}/members")
        if data:
            members = []
            for member in data:
                m = Member.from_dict(member)
                members.append(m)
            return members

    @classmethod
    async def leaderboard(cls, id, session, limit=10):
        data = await session.get(f"guilds/{id}/leaderboard?limit={limit}")
        if data:
            members = []
            for member in data:
                m = Member.from_dict(member)
                members.append(m)
            return members

    @property
    def id(self):
        return self.guild_id

    @id.setter
    def id(self, value):
        raise Exception("id is a read-only property!")

    @property
    def prefix(self):
        return self.ensure_setting(constants.GuildSettings.prefix, "$")

    @prefix.setter
    def prefix(self, new_prefix):
        self.set_setting(constants.GuildSettings.prefix, new_prefix)

    @property
    def settings(self):
        """Return Guild settings as a JSONDict."""
        return self._settings

    @settings.setter
    def settings(self, new_settings: dict):
        """Set Guild settings as `dict`."""
        self._settings = new_settings

    async def save(self, session):
        await session.patch("guilds/" + str(self.guild_id), self.to_dict())

    def get_setting(self, setting_key):
        """Gets the value at `setting_key`. If it doesn't exist then returns `None`."""
        _path = setting_key.split(".")
        _result = self.settings
        for x in _path:
            _result = _result.get(x, {})
            if _result == {}:
                return None
        return _result

    def get_raw_settings(self) -> str:
        """Return string of guild settings."""
        return self.settings

    def set_setting(self, setting_key, value):
        """Adds/Overwrites the current `setting_key` with `value`
        If `setting_key` doesn't exist then it is created.

        Returns `value`.
        """
        _lst = self.settings
        self._nested_set(_lst, setting_key.split("."), value)
        self.settings = json.dumps(_lst)
        return value

    def _nested_set(self, dic, keys, value):
        for key in keys[:-1]:
            dic = dic.setdefault(key, {})
        dic[keys[-1]] = value

    def ensure_setting(self, setting_key, value):
        """Ensure that `setting_key` exists, if not then set's default to `value`."""
        result = self.get_setting(setting_key)
        if result is None:
            result = self.set_setting(setting_key, value)
        return result

    def __str__(self):
        return "<KatGuild (id={}, prefix={}, settings={})>".format(
            self.guild_id, self.prefix, self.settings
        )

    def to_dict(self):
        # TODO: For some reason self.settings here is a str instead
        # of a dict. I have no idea why... For now we just json.loads it.
        return {"id": self.id, "settings": json.loads(self.settings)}

class User:
    """User information from Kat API."""

    def __init__(self, id, birthday="None", birthday_years=0):
        self.user_id: int = id
        self.birthday_years: int = birthday_years
        self.birthday = (
            datetime.datetime.strptime(birthday, "%Y-%m-%d")
            if birthday != "None"
            else None
        )

    @classmethod
    def from_dict(cls, data):
        return cls(
            data["id"],
            birthday=data.get("birthday", None),
            birthday_years=data.get("years", 0),
        )

    def to_json(self):
        return self.__repr__()

    @property
    def id(self):
        return self.user_id

    @id.setter
    def id(self, value):
        raise Exception("id is a read-only property!")

    @classmethod
    async def get(cls, id, session):
        data = await session.get("users/" + str(id))
        if data[0]:
            return cls.from_dict(data["data"][0])
        else:
            return cls.from_dict({"id": id})

    async def save(self, session):
        await session.patch("users/" + str(self.user_id), self.to_json())

    def __str__(self):
        return "<User (id={})>".format(self.user_id)

    def __repr__(self):
        return {
            "id": self.user_id,
            "birthday": self.birthday.strftime("%Y-%m-%d") if self.birthday else None,
            "years": self.birthday_years,
        }

class Member:
    """Member information from Kat DB.

    `guild_id`:int  ;Guild ID of which the member belongs to

    `user_id`:int   ;ID of the user of which the member belongs to

    `_data`:dict    ;JSON dict of member-data. (for now just includes warning system data.
    """

    def __init__(self, gid, uid, lvl, xp, settings=""):
        self.guild_id = gid
        self.user_id = uid
        self._settings = settings

        self.xp = xp
        self.lvl = lvl

    @classmethod
    def from_dict(cls, data):
        return cls(
            data["gid"],
            data["id"],
            data.get("level", 0),
            data.get("xp", 0),
            data.get("settings", {}),
        )

    @classmethod
    async def get(cls, gid, uid, session):
        data = await session.get(f"guilds/{gid}/{uid}")
        if data[0]:
            return cls.from_dict(data[0])
        else:
            return cls.from_dict({"gid": gid, "id": uid, "level": 0, "xp": 0})

    @property
    def id(self):
        return (self.guild_id, self.user_id)

    @id.setter
    def id(self, value):
        raise Exception("id is a read-only property!")

    def set_xp(self, value):
        self.xp = value

    @property
    def settings(self):
        """Return Guild settings as a JSONDict."""
        return self._settings

    @settings.setter
    def settings(self, new_settings: dict):
        """Set Guild settings as `str`."""
        self._settings = new_settings

    def get_setting(self, setting_key):
        """Gets the value at `setting_key`. If it doesn't exist then returns `None`."""
        _path = setting_key.split(".")
        _result = self.settings
        for x in _path:
            _result = _result.get(x, {})
            if _result == {}:
                return None
        return _result

    def get_raw_settings(self) -> str:
        """Return string of member settings."""
        return self.settings

    def set_setting(self, setting_key, value):
        """Adds/Overwrites the current `setting_key` with `value`
        If `setting_key` doesn't exist then it is created.

        Returns `value`.
        """
        _lst = self.settings
        self._nested_set(_lst, setting_key.split("."), value)
        self.settings = json.dumps(_lst)
        return value

    def _nested_set(self, dic, keys, value):
        for key in keys[:-1]:
            dic = dic.setdefault(key, {})
        dic[keys[-1]] = value

    def ensure_setting(self, setting_key, value):
        """Ensure that `setting_key` exists, if not then set's default to `value`."""
        result = self.get_setting(setting_key)
        if result is None:
            result = self.set_setting(setting_key, value)
        return result

    async def save(self, session):
        await session.patch(f"guilds/{self.guild_id}/{self.user_id}", self.__repr__())

    def __str__(self):
        return "<Member (guild_id={}, user_id={}, (xp={},lvl={}))>".format(
            self.guild_id, self.user_id, self.xp, self.lvl
        )

    def __repr__(self):
        return {
            "id": self.user_id,
            "gid": self.guild_id,
            "level": self.lvl,
            "xp": self.xp,
            "settings": self.settings,
        }


from datetime import timedelta
import boto3
import os
from io import StringIO, BytesIO
import feedparser
import pandas as pd
import spacy
import requests
import pybase64
from google.cloud import bigquery, storage
import shutil

from glob import glob
# The DAG object; we'll need this to instantiate a DAG
from airflow import DAG
from airflow.decorators import task
# from airflow.utils.dates import days_ago
from airflow.operators.empty import EmptyOperator
from airflow.operators.bash import BashOperator
from airflow.models import Variable
# from airflow.providers.discord.operators.discord_webhook import DiscordWebhookOperator
from datetime import datetime

import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer

from utils import upload_formatted_rss_feed,scrape_reviews,scrape_appdetails,get_unique_appids,analyze_sentiment,label_polarity,upload_string_to_gcs,webhook_message

BUCKET_NAME = "news_sites"
DATE_NOW = datetime.now().strftime("%Y-%m-%d")
# Data directory for CSVs and OSM Images
DATA_PATH = '/opt/airflow/data/'

GAME_ARTICLE_FILENAME = f"{DATA_PATH}game_articles_{DATE_NOW}.csv"
GAME_REVIEWS_FILENAME = f"{DATA_PATH}game_reviews_{DATE_NOW}.csv"
GAME_DETAILS_FILENAME = f"{DATA_PATH}game_details_{DATE_NOW}.csv"

##################################################################################################################################
################################################# 1.) EXTRACT #################################################################
##################################################################################################################################
@task(task_id="kotaku")
def kotaku_feed(ds=None, **kwargs):
    upload_formatted_rss_feed("kotaku","https://kotaku.com/rss")

@task(task_id="escapist_mag")
def escapist_mag_feed(ds=None, **kwargs):
    upload_formatted_rss_feed("escapist_mag","https://www.escapistmagazine.com/v2/feed/")

@task(task_id="eurogamer")
def eurogamer_feed(ds=None, **kwargs):
    upload_formatted_rss_feed("eurogamer","https://www.eurogamer.net/?format=rss")

@task(task_id="indigames_plus")
def indigames_plus_feed(ds=None, **kwargs):
    upload_formatted_rss_feed("indigames_plus","https://indiegamesplus.com/feed")

@task(task_id="rock_paper_sg")
def rock_paper_sg_feed(ds=None, **kwargs):
    upload_formatted_rss_feed("rock_paper_sg","http://feeds.feedburner.com/RockPaperShotgun")

@task(task_id="ancient_gaming")
def ancient_gaming_feed(ds=None, **kwargs):
    upload_formatted_rss_feed("ancient_gaming","http://feeds.feedburner.com/TheAncientGamingNoob")

@task(task_id="combine_all_articles")
def combine_all_articles(ds=None,**kwargs):
    files = os.listdir(f"{DATA_PATH}articles")
    dfs = []
    for file in files:
        outfile = f"{DATA_PATH}articles/{file}"
        if not outfile.endswith('.csv'):
            continue
        df = pd.read_csv(outfile,on_bad_lines="skip")
        dfs.append(df)
    game_articles = pd.concat(dfs)
    game_articles = game_articles.drop_duplicates()
    game_articles = game_articles.dropna(subset=["appids"])
    game_articles.to_csv(GAME_ARTICLE_FILENAME,index=False)

@task(task_id="scrape_game_reviews")
def scrape_game_reviews(ds=None,**kwargs):
    game_articles = pd.read_csv(GAME_ARTICLE_FILENAME,on_bad_lines="skip")
    appids = get_unique_appids(game_articles)

    all_reviews = []
    for appid in appids:
        reviews_list = scrape_reviews(appid)
        all_reviews += reviews_list
    game_reviews = pd.DataFrame(all_reviews)

    game_reviews = game_reviews.drop_duplicates(subset=["review"])

    game_reviews["timestamp_created"] = game_reviews["timestamp_created"].apply(lambda x: datetime.utcfromtimestamp(int(x)).strftime("%Y-%m-%d %H:%M:%S"))
    game_reviews["timestamp_updated"] = game_reviews["timestamp_updated"].apply(lambda x: datetime.utcfromtimestamp(int(x)).strftime("%Y-%m-%d %H:%M:%S"))
    game_reviews.to_csv(GAME_REVIEWS_FILENAME,index=False)

@task(task_id="scrape_game_details")
def scrape_game_details(ds=None,**kwargs):
    game_articles = pd.read_csv(GAME_ARTICLE_FILENAME,on_bad_lines="skip")
    appids = get_unique_appids(game_articles)

    lst_game_details = scrape_appdetails(appids)

    game_details = pd.DataFrame(lst_game_details).rename(columns={"appid":"appids"})
    game_details = game_details.drop_duplicates(subset=["appids"])
    game_details.to_csv(GAME_DETAILS_FILENAME,index=False)

##################################################################################################################################
################################################# 2.) TRANSFORM  #################################################################
##################################################################################################################################

@task(task_id="word_count")
def word_count(ds=None, **kwargs):

    def word_count(text):
        words = text.split()
        freq = [words.count(w) for w in words]
        word_dict = dict(zip(words, freq))
        return word_dict

    game_articles = pd.read_csv(GAME_ARTICLE_FILENAME,on_bad_lines="skip")
    game_reviews = pd.read_csv(GAME_REVIEWS_FILENAME,on_bad_lines="skip")

#############################################################################################
    print(f"getting word_count for game articles...")
    game_articles['sum_word_cnt'] = game_articles['summary'].apply(lambda x: len(str(x).split()))
    game_articles['dict_word_cnt'] = game_articles['summary'].apply(lambda x: word_count(str(x)))
    game_articles.to_csv(GAME_ARTICLE_FILENAME, index=False)
    print(f"word_count for game_articles collected!")
    print()

#############################################################################################
    print(f"getting word_count for game reviews...")
    game_reviews['sum_word_cnt'] = game_reviews['review'].apply(lambda x: len(str(x).split()))
    game_reviews['dict_word_cnt'] = game_reviews['review'].apply(lambda x: word_count(str(x)))

    game_reviews.to_csv(GAME_REVIEWS_FILENAME,index=False)
    print(f"word_count for game reviews collected!")
    print()

@task(task_id='sentiment_analysis')
def sentiment_analysis(ds=None,**kwargs):

    game_articles = pd.read_csv(GAME_ARTICLE_FILENAME,on_bad_lines="skip")
    game_reviews = pd.read_csv(GAME_REVIEWS_FILENAME,on_bad_lines="skip")

    print("getting sentiment for articles...")
    game_articles['polarity_score_summary'] = game_articles['summary'].apply(lambda x: analyze_sentiment(str(x)))
    game_articles["sentiment_summary"] = game_articles["polarity_score_summary"].apply(lambda x: label_polarity(x["compound"]))

    game_articles['polarity_score_title'] = game_articles['title'].apply(lambda x: analyze_sentiment(str(x)))
    game_articles["sentiment_title"] = game_articles["polarity_score_title"].apply(lambda x: label_polarity(x["compound"]))
    game_articles.to_csv(GAME_ARTICLE_FILENAME, index=False)
    print("sentiment for articles taken!")

    #############################################################################################
    print("getting sentiment for game reviews...")
    game_reviews["sentiment"] = game_reviews["voted_up"].apply(lambda x: "positive" if x else "negative")
    game_reviews.to_csv(GAME_REVIEWS_FILENAME,index=False)
    print("sentiment for reviews taken!")
# NER
@task(task_id='spacy_ner')
def spacy_ner(ds=None, **kwargs):
    nlp = spacy.load("/model/en_core_web_sm/en_core_web_sm-3.3.0")

    def ner(text):
        doc = nlp(text)
        ner = {}
        for entity in doc.ents:
            ner[entity.text] = entity.label_
            print(entity.text, entity.label_)
        return ner

    game_articles = pd.read_csv(GAME_ARTICLE_FILENAME,on_bad_lines="skip")
    game_reviews = pd.read_csv(GAME_REVIEWS_FILENAME,on_bad_lines="skip")

    print("getting NER for game articles...")
    game_articles['NER'] = game_articles['summary'].apply(lambda x: ner(str(x)))
    game_articles.to_csv(GAME_ARTICLE_FILENAME,index=False)
    print("getting NER for game articles complete!")
    print()
    ###########################################################################################

    print("getting NER for game reviews...")
    game_reviews["NER"] = game_reviews["review"].apply(lambda x: ner(str(x)))
    game_reviews.to_csv(GAME_REVIEWS_FILENAME,index=False)
    print("getting NER for game reviews complete!")
    print()

##################################################################################################################################
################################################# 3.) LOAD  ######################################################################
##################################################################################################################################

@task(task_id="load_data")
def load_data(ds=None, **kwargs):
    files = os.listdir(DATA_PATH)
    for file in files:
        outfile = f"{DATA_PATH}{file}"
        if not outfile.endswith('.csv'):
            continue
        df = pd.read_csv(outfile,on_bad_lines="skip")
        csv_buffer = StringIO()

        if "details" in file:
            filename = f"game_details/{file}"
        elif "articles" in file:
            filename = f"game_articles/{file}"
        else:
            filename = f"game_reviews/{file}"

        df.to_csv(csv_buffer,index=False)
        upload_string_to_gcs(csv_body=csv_buffer, uploaded_filename=filename)

@task(task_id='delete_residuals')
def delete_residuals(ds=None, **kwargs):
    files = os.listdir(f"{DATA_PATH}")
    for file in files:
        outfile = f"{DATA_PATH}{file}"
        print(file)
        if os.path.isdir(outfile):
            shutil.rmtree(outfile)
        else:
            os.remove(outfile)

@task(task_id="data_validation")
def data_validation(ds=None,**kwargs):
    game_articles = pd.read_csv(GAME_ARTICLE_FILENAME)
    game_reviews = pd.read_csv(GAME_REVIEWS_FILENAME)
    game_details = pd.read_csv(GAME_DETAILS_FILENAME)

    game_details = game_details.dropna(subset=["name"])
    game_details = game_details.rename(columns={"appid":"appids"})
    game_details = game_details.drop_duplicates(subset=["appids"])
    game_details.to_csv(GAME_DETAILS_FILENAME,index=False)

    game_reviews = game_reviews.dropna(subset=["review"])
    game_reviews = game_reviews.drop_duplicates(subset=["review"])
    game_reviews.to_csv(GAME_REVIEWS_FILENAME,index=False)

    game_articles = game_articles.dropna(subset=["summary"])
    game_articles = game_articles.drop_duplicates()
    game_articles.to_csv(GAME_ARTICLE_FILENAME,index=False)

##################################################################################################################################
################################################# DAG ############################################################################
##################################################################################################################################

with DAG(
    'scrapers_proj_test',
    default_args={

    },
    description='Pipeline demo',
    schedule_interval=timedelta(days=1),
    start_date=datetime(2022, 6, 25),
    catchup=False,
    tags=['scrapers'],
) as dag:

    dagstart_msg = BashOperator(
        task_id="dagstart_msg",
        bash_command = webhook_message("Dag tasks starting!"),
        dag=dag
    )

    article_scraping1 = BashOperator(
        task_id="article_scraping1",
        bash_command = webhook_message("task 1.0, initiating data extraction: article scraping 1!"),
        dag=dag
    )

    article_scraping1_end = BashOperator(
        task_id="article_scraping1_end",
        bash_command = webhook_message("continuing task 1.0: article scraping 1 complete!"),
        dag=dag
    )

    article_scraping2 = BashOperator(
        task_id="article_scraping2",
        bash_command = webhook_message("continuing task 1.0: article scraping 2!"),
        dag=dag
    )

    article_scraping2_end = BashOperator(
        task_id="article_scraping2_end",
        bash_command = webhook_message("ending task 1.0: article scraping complete!"),
        dag=dag
    )

    steam_scraping = BashOperator(
        task_id="steam_scraping",
        bash_command = webhook_message("starting task 1.5: scraping user reviews and game details!"),
        dag=dag
    )

    steam_scraping_end = BashOperator(
        task_id="data_extraction_end",
        bash_command = webhook_message("ending task 1.5: steam scraping complete! all data extracted."),
        dag=dag
    )

    transformation = BashOperator(
        task_id="transformation_start",
        bash_command = webhook_message("starting task 2.0: beginning data transformation"),
        dag=dag
    )
    transformation_end = BashOperator(
        task_id="transformation_end",
        bash_command = webhook_message("ending task 2.0: data transformation completed!"),
        dag=dag
    )

    loading_data = BashOperator(
        task_id="data_loading",
        bash_command = webhook_message("starting task 3.0: beginning data loading"),
        dag=dag
    )
    loading_data_end = BashOperator(
        task_id="data_loading_end",
        bash_command = webhook_message("ending task 3.0: data successfully loaded to the cloud!"),
        dag=dag
    )
    dagend_msg = BashOperator(
        task_id="dagend_msg",
        bash_command = webhook_message("All dag tasks completed!"),
        dag=dag
    )

    dagstart_msg \
    >> article_scraping1 >> [indigames_plus_feed(),kotaku_feed(), escapist_mag_feed()] >> article_scraping1_end \
    >> article_scraping2 >> [eurogamer_feed(),rock_paper_sg_feed(),ancient_gaming_feed()] >> combine_all_articles() >> article_scraping2_end \
    >> steam_scraping >> [scrape_game_details(),scrape_game_reviews()] >> steam_scraping_end \
    >> transformation >> sentiment_analysis() >> word_count() >> spacy_ner() >> transformation_end >> data_validation() \
    >> loading_data >> load_data() >> loading_data_end >> delete_residuals() >> \
    dagend_msg


import pathlib
import unittest

import onnx
from onnx import TensorProto, helper, shape_inference

from ..mobile_helpers.usability_checker import check_shapes
from ..onnx_model_utils import (
    fix_output_shapes,
    get_producer_consumer_maps,
    is_fixed_size_tensor,
    make_dim_param_fixed,
    make_input_shape_fixed,
)

script_dir = pathlib.Path(__file__).parent
ort_root = script_dir.parents[3]

# example usage from <ort root>/tools/python
# python -m unittest util/test/test_onnx_model_utils.py
# NOTE: at least on Windows you must use that as the working directory for all the imports to be happy

class TestGetProducerConsumerMaps(unittest.TestCase):
    @staticmethod
    def _create_model():
        # create a model with subgraphs and various types of shadowing
        body = helper.make_graph(
            [
                # shadow a1 in main graph.
                # LoopAdd_SubgraphOutput should be linked to this and a1 should not be an implicit input
                helper.make_node("Add", ["loop_state_in", "iter"], ["a1"], "LoopAdd_Shadows"),
                # main_graph_initializer should be handled (implicit input but no producer node)
                # graph input 'x' from main graph should also be handled
                helper.make_node("Add", ["main_graph_initializer", "x"], ["a2"], "LoopAdd_OuterScopeInitializer"),
                # implicit input should be handled - 'z' can be accessed from outside scope
                # Add2 in main graph should be implicit input of the Loop node
                helper.make_node("Add", ["z", "a1"], ["a3"], "LoopAdd_ImplicitInput"),
                # create subgraph output
                helper.make_node("Add", ["a2", "a3"], ["loop_state_out"], "LoopAdd_SubgraphOutput"),
            ],
            "Loop_body",
            [
                helper.make_tensor_value_info("iter", TensorProto.INT64, [1]),
                helper.make_tensor_value_info("cond", TensorProto.BOOL, [1]),
                helper.make_tensor_value_info("loop_state_in", TensorProto.FLOAT, [1]),
            ],
            [
                helper.make_tensor_value_info("cond", TensorProto.BOOL, [1]),
                helper.make_tensor_value_info("loop_state_out", TensorProto.FLOAT, [1]),
            ],
            [],
        )

        # Create the main graph
        graph_proto = helper.make_graph(
            [
                # create 'a1' which is shadowed in the subgraph. node should not be joined to Loop1
                helper.make_node("Add", ["x", "y"], ["a1"], "Add1"),
                # create 'z' which is an implicit input to subgraph. node should be joined to Loop1
                helper.make_node("Add", ["a1", "main_graph_initializer"], ["z"], "Add2"),
                # rename 'z' to use as explicit input to Loop
                helper.make_node("Identity", ["z"], ["state_var_in"], "RenameZ"),
                helper.make_node(
                    "Loop", ["max_trip_count", "keep_going", "state_var_in"], ["state_var_out"], "Loop1", body=body
                ),
                helper.make_node("Sub", ["a1", "state_var_out"], ["graph_output"], "sub_1"),
            ],
            "Main_graph",
            [
                helper.make_tensor_value_info("x", TensorProto.FLOAT, [1]),
                helper.make_tensor_value_info("y", TensorProto.FLOAT, [1]),
            ],
            [
                helper.make_tensor_value_info("graph_output", TensorProto.FLOAT, [1]),
            ],
            [
                helper.make_tensor("max_trip_count", TensorProto.INT64, [1], [2]),
                helper.make_tensor("main_graph_initializer", TensorProto.FLOAT, [1], [1.0]),
                helper.make_tensor("keep_going", TensorProto.BOOL, [1], [True]),
            ],
        )

        return helper.make_model(graph_proto)

    def test_model_with_subgraph(self):
        """
        Test a manually created model that has a subgraph and implicit inputs of all possible types.
        """

        model = self._create_model()
        node_to_producers, node_to_consumers = get_producer_consumer_maps(model.graph)

        main_graph_add_create_a1 = model.graph.node[0]
        main_graph_add_create_z = model.graph.node[1]
        main_graph_rename_z = model.graph.node[2]
        main_graph_loop = model.graph.node[3]
        main_graph_sub = model.graph.node[4]

        subgraph = main_graph_loop.attribute[0].g
        loop_add_shadow = subgraph.node[0]
        loop_add_outer_scope_init = subgraph.node[1]
        loop_add_implicit_input = subgraph.node[2]
        loop_add_subgraph_output = subgraph.node[3]

        def node_name(node):
            return f"{node.name}:{node.op_type}"

        def check_linked(producer, consumer):
            self.assertTrue(
                producer in node_to_producers[consumer],
                f"{node_name(producer)} not in producers for {node_name(consumer)}",
            )
            self.assertTrue(
                consumer in node_to_consumers[producer],
                f"{node_name(consumer)} not in consumers for {node_name(producer)}",
            )

        def check_not_linked(producer, consumer):
            self.assertFalse(
                producer in node_to_producers[consumer], f"{node_name(producer)} in producers for {node_name(consumer)}"
            )
            self.assertFalse(
                consumer in node_to_consumers[producer],
                f"{node_name(consumer)} not in consumers for {node_name(producer)}",
            )

        check_linked(main_graph_add_create_a1, main_graph_add_create_z)
        # a1 in main graph shouldn't be implicit input to loop as it is shadowed
        check_not_linked(main_graph_add_create_a1, main_graph_loop)
        # z is implicit input
        check_linked(main_graph_add_create_z, main_graph_loop)
        check_linked(main_graph_rename_z, main_graph_loop)
        check_linked(main_graph_loop, main_graph_sub)

        # check subgraph
        check_linked(loop_add_shadow, loop_add_implicit_input)
        check_linked(loop_add_outer_scope_init, loop_add_subgraph_output)
        check_linked(loop_add_implicit_input, loop_add_subgraph_output)

class TestDynamicDimReplacement(unittest.TestCase):
    def test_replace_symbolic_dim(self):
        """
        Update a model with a single symbolic input dimension. After replacement run shape inferencing to verify that
        all shapes in the model have fixed sizes.
        """
        model_path = (
            ort_root / "onnxruntime" / "test" / "testdata" / "CNTK" / "test_LSTM.tanh.bidirectional" / "model.onnx"
        )

        model = onnx.load_model(str(model_path))

        # validate the expected input after inferring shape info
        m2 = shape_inference.infer_shapes(model, True)
        dynamic_inputs, num_dynamic_values = check_shapes(m2.graph)
        self.assertEqual(len(dynamic_inputs), 1)
        self.assertEqual(dynamic_inputs[0].name, "Input3")
        self.assertGreater(num_dynamic_values, 0)

        # update original model
        make_dim_param_fixed(model.graph, "None", 4)

        # and validate the model no longer has dynamic values
        model = shape_inference.infer_shapes(model, True)
        dynamic_inputs, num_dynamic_values = check_shapes(model.graph)
        self.assertFalse(dynamic_inputs)
        self.assertEqual(num_dynamic_values, 0)

    def test_replace_input_shape(self):
        """
        Replace the entire shape for an input. This can be used when the model has inputs with unknown dimensions
        i.e. the dimension has no value and no symbolic name so it's harder to replace.
        """
        model_path = ort_root / "onnxruntime" / "test" / "testdata" / "gh_issue_9671.onnx"

        model = onnx.load_model(str(model_path))

        # validate the expected input after inferring shape info
        m2 = shape_inference.infer_shapes(model, True)
        dynamic_inputs, num_dynamic_values = check_shapes(m2.graph)
        self.assertEqual(len(dynamic_inputs), 3)
        self.assertEqual(dynamic_inputs[0].name, "X1")
        self.assertEqual(dynamic_inputs[1].name, "X2")
        self.assertEqual(dynamic_inputs[2].name, "X3")
        self.assertGreater(num_dynamic_values, 0)

        # update original model
        make_input_shape_fixed(model.graph, "X1", [2, 2, 4])
        make_input_shape_fixed(model.graph, "X2", [2, 4])
        make_input_shape_fixed(model.graph, "X3", [2, 2, 4])

        # and validate the model no longer has dynamic values
        model = shape_inference.infer_shapes(model, True)
        dynamic_inputs, num_dynamic_values = check_shapes(model.graph)
        self.assertFalse(dynamic_inputs)

    def test_replace_input_shape_with_dim_params(self):
        # replace the input shape where the existing shape also has dim_param entries.
        # in this case we should also iterate the rest of the model and replace other instances
        # of the dim_param with the new value.
        model_path = ort_root / "onnxruntime" / "test" / "testdata" / "fuse_mul_1.onnx"
        model = onnx.load_model(str(model_path))

        m2 = shape_inference.infer_shapes(model, True)
        dynamic_inputs, num_dynamic_values = check_shapes(m2.graph)
        self.assertEqual(len(dynamic_inputs), 1)
        self.assertEqual(dynamic_inputs[0].name, "X1")
        # input as well as other values in model have shape ['D'] so check > 1
        self.assertGreater(num_dynamic_values, 1)

        # replace X1's shape of ['D'] -> [4]
        make_input_shape_fixed(model.graph, "X1", [4])

        # validate the model no longer has dynamic values
        # we don't run shape_inference here as 'D' is the only dimension in the whole model, and we should have
        # replaced every instance of it if _make_input_shape_fixed worked as expected.
        model = shape_inference.infer_shapes(model, True)
        dynamic_inputs, num_dynamic_values = check_shapes(model.graph)
        self.assertFalse(dynamic_inputs)
        self.assertEqual(num_dynamic_values, 0)

    def test_fix_output_shape(self):
        """
        Replace an input shape in a model where that won't update the output shape automatically.
        Manually fix the output so the usage of the model is clearer.
        """
        model_path = ort_root / "onnxruntime" / "test" / "testdata" / "transform" / "fusion" / "bias_gelu_fusion.onnx"
        model = onnx.load_model(str(model_path))

        make_input_shape_fixed(model.graph, "A", [2, 2, 3072])

        # symbolic dim names in graph inputs don't match graph outputs so they won't have been updated yet
        self.assertFalse(is_fixed_size_tensor(model.graph.output[0]))
        fix_output_shapes(model)
        self.assertTrue(is_fixed_size_tensor(model.graph.output[0]))

    def test_invalid_replace_input_shape(self):
        model_path = ort_root / "onnxruntime" / "test" / "testdata" / "sklearn_bin_voting_classifier_soft.onnx"
        model = onnx.load_model(str(model_path))
        # test some invalid usages
        self.assertRaisesRegex(
            ValueError,
            "Rank mismatch. Existing:2 Replacement:3",
            make_input_shape_fixed,
            model.graph,
            "input",
            [1, 2, 3],
        )

        self.assertRaisesRegex(
            ValueError,
            "Can't replace existing fixed size of 2 with 3 for dimension 2",
            make_input_shape_fixed,
            model.graph,
            "input",
            [4, 3],
        )

        self.assertRaisesRegex(
            ValueError, "Input X1 was not found in graph inputs.", make_input_shape_fixed, model.graph, "X1", [2, 3]
        )


import pandas
import statistics
import numpy as np
from pandas.plotting import scatter_matrix
import matplotlib.pyplot as plt
from sklearn import model_selection
from sklearn import datasets, linear_model
from sklearn import svm
from sklearn.metrics import mean_squared_error, r2_score, explained_variance_score

# Function: returns error values for model test
def getErrorValues(pred,yVal):
	count = 0
	diffArray = []
	for i in range (0,len(pred)):
		value = pred[i]
		diff = abs(value-yVal[i])
		diffArray.append(diff)
		count += diff
	meanError = count/len(pred)
	medianError = statistics.median(diffArray)
	return meanError, medianError

# Function: tests accuracy of potential model
def train_model(X,Y,numTrials,printThreshold):
	p = len(X[0])
	meanErrors = []
	medianErrors = []
	averageMSE = 0
	averageR2 = 0
	averageEVS = 0
	N = 0
	for i in range (0,numTrials):
		X_train, X_validation, Y_train, Y_validation = model_selection.train_test_split(X, Y, test_size=0.2, random_state=i*1319)
		RR.fit(X_train, Y_train)
		predictions = RR.predict(X_validation)
		meanErrors.append(0)
		medianErrors.append(0)
		meanErrors[i], medianErrors[i] = getErrorValues(predictions,Y_validation)
		averageMSE += mean_squared_error(Y_validation, predictions)
		averageR2 += r2_score(Y_validation, predictions)
		averageEVS += explained_variance_score(Y_validation, predictions)
		N = len(predictions)
	trials = float(len(meanErrors))
	R2 = averageR2/trials
	adjR2 = 1 - (((1 - R2)*(N-1))/(N-p-1))
	print (adjR2)
	if (adjR2 > printThreshold):
		print ("Average MSE: " + str(averageMSE/trials))
		print ("Average Variance (R^2): " + str(R2))
		print ("Average Adjusted R^2: " + str(adjR2))
		print ("Average Explained Variance Score: " + str(averageEVS/trials))
		print ("Average Mean-Error: " + str(sum(meanErrors)/trials))
		print ("Average Median-Error: " + str(sum(medianErrors)/trials))
		print ("Median Mean-Error: " + str(statistics.median(meanErrors)))
		print ("Median Median-Error: " + str(statistics.median(medianErrors)))

# Functions below are used to test models for a specific statistic
def train_goals():
	print ("Goals:")
	X = arrayX[:,[4,6,10,19,57,61,73,88]]
	Y = arrayY[:,9]
	train_model(X,Y,2000,0.0)
	print (" ")
	#0.626

def train_gpg():
	print ("Goals per Game:")
	X = arrayX[:,[4,6,10,26,57,69,82,88]]
	#X = arrayX[:,[4,6,10,19,57,69,73,88]]
	Y = arrayY[:,10]
	train_model(X,Y,2000,0.0)
	print (" ")
	#0.657

def train_assists():
	print ("Assists:")
	X = arrayX[:,[4,13,19,59,80]]
	Y = arrayY[:,11]
	train_model(X,Y,2000,0.0)
	print (" ")
	#0.611

def train_apg():
	print ("Assists per Game:")
	#X = arrayX[:,[4,13,59,90,93]]
	X = arrayX[:,[4,6,13,43,59,93]]
	Y = arrayY[:,12]
	train_model(X,Y,2000,0.0)
	print (" ")
	#0.642

def train_powerplaypoints():
	print ("Powerplay:")
	X = arrayX[:,[4,14,56,64,66]]
	Y = arrayY[:,16]
	train_model(X,Y,2000,0.0)
	print (" ")
	#0.645

def train_pppg():
	print ("Powerplay Points per Game:")
	#X = arrayX[:,[4,14,29,64,66]]
	X = arrayX[:,[4,14,17,29,61,63,73]]
	Y = arrayY[:,17]
	train_model(X,Y,2000,0.0)
	print (" ")
	#0.663

def train_plusminus():
	print ("Plus/Minus:")
	#X = arrayX[:,[62,84,92]]
	X = arrayX[:,[33,37,62,67]]
	Y = arrayY[:,15]
	train_model(X,Y,2000,0.0)
	print (" ")
	#0.057

def train_shots():
	print ("Shots:")
	X = arrayX[:,[4,13,20,66]]
	Y = arrayY[:,19]
	train_model(X,Y,2000,0.0)
	print (" ")
	#0.687

def train_spg():
	print ("Shots per Game:")
	X = arrayX[:,[4,6,14,20,26,67]]
	Y = arrayY[:,20]
	train_model(X,Y,2000,0.0)
	print (" ")
	#0.784

def train_hits():
	print ("Hits:")
	X = arrayX[:,[21,22,91]]
	Y = arrayY[:,21]
	train_model(X,Y,2000,0.0)
	print (" ")
	#0.669

def train_hpg():
	print ("Hits per Game:")
	X = arrayX[:,[22,61,69]]
	Y = arrayY[:,22]
	train_model(X,Y,2000,0.0)
	print (" ")
	#0.791

def train_blocks():
	print ("Blocks:")
	X = arrayX[:,[6,23,25,27,71]]
	Y = arrayY[:,23]
	train_model(X,Y,2000,0.0)
	print (" ")
	#0.755

def train_bpg():
	print ("Blocks per Game:")
	#X= arrayX[:,[6,24,71,90,91]]
	X = arrayX[:,[6,24,71,85,91]]
	Y = arrayY[:,24]
	train_model(X,Y,2000,0.0)
	print (" ")
	#0.874

#Function: Used to test models
def train_AModel():
	#X = arrayX[:,[22,61,69]]
	#0.791

	indexs = [91]
	#indexs = [4,6,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,51,53,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93]
	for j in range (0,len(indexs)):
		print (indexs[j])
		X = arrayX[:,[22,61,69]]
		Y = arrayY[:,22]
		train_model(X,Y,200,0.792)
	#0.6629/0.6633

url = "Skaters20132016C.csv"
ds1316 = pandas.read_csv(url)
url = "Skaters20142017C.csv"
ds1417 = pandas.read_csv(url)
url = "Skaters20152018C.csv"
ds1518 = pandas.read_csv(url)
url = "Skaters20152016C.csv"
ds1516 = pandas.read_csv(url)
url = "Skaters20162017C.csv"
ds1617 = pandas.read_csv(url)
url = "Skaters20172018C.csv"
ds1718 = pandas.read_csv(url)

# Split-out validation dataset
array1516 = ds1516.values
array1617 = ds1617.values
array1718 = ds1718.values
array1316 = ds1316.values
array1417 = ds1417.values

arrayX1 =  np.concatenate((array1516, array1316), axis=1)
arrayX2 =  np.concatenate((array1617, array1417), axis=1)
arrayX =  np.concatenate((arrayX1, arrayX2), axis=0)

arrayY = np.concatenate((array1617, array1718), axis=0)
LR = linear_model.LinearRegression()
RR = linear_model.Ridge(alpha = .5)
SVR = svm.SVR()
#train_goals()
#train_gpg()
#train_assists()
#train_apg()
#train_powerplaypoints()
#train_pppg()
#train_plusminus()
#train_shots()
#train_spg()
#train_hits()
#train_hpg()
#train_blocks()
#train_bpg()
train_AModel()


#from app.api.api_v1.router import api1_router
from app.api.router import api_router
from fastapi import FastAPI
from app.core import config

app = FastAPI(title=config.PROJECT_NAME,
              description=config.PROJECT_DESCRIPTION,
              version=config.PROJECT_VERSION,
              openapi_url=f"{config.API_VERSION_STR}/openapi.json",
              docs_url=config.DOCS_URL,
              redoc_url=config.REDOC_URL)

app.include_router(api_router)
#app.include_router(api1_router, prefix=config.API_VERSION_STR)


import math
import smtplib
import random
import string

from email.mime.text import MIMEText
from email.header import Header
from django.conf import settings

#Utililty/ reusable functions:

def generate_token():
	return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(settings.TOKEN_SIZE))

def divide_list_by_two (whole_list):
    half_size_ceil = int(math.ceil(len(whole_list)/2))
    half_size_floor = int(math.floor(len(whole_list)/2))
    list_1 = whole_list [:half_size_ceil]
    list_2 = whole_list [len(whole_list) - half_size_floor:]

    return {"list_1": list_1, "list_2": list_2}

def send_email(to_address, msg):
  msg = MIMEText(msg)
  msg['Subject'] = "EasyDjango!"
  msg['From']    = settings.SMTP_EMAIL_SENDER
  msg['To']      = to_address

  s = smtplib.SMTP(settings.SMTP_HOST, settings.SMTP_HOST_PORT)

  s.login(settings.SMTP_SERVER_LOGIN, settings.SMTP_SERVER_PASSWORD)
  s.sendmail(msg['From'], msg['To'], msg.as_string())
  s.quit()


#-*- coding: utf-8 -*-
'''
codigo ejemplo llamada a templates
'''

from flask import Flask
from flask import render_template

app = Flask(__name__)

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/user')
@app.route('/user/<name>')
def user( name='usuario'):
    #name='Cris'
    age=48
    lista=[1,2,3,4]
    return render_template('user.html', name=name, age=age, lista=lista)

if __name__ == '__main__':
    app.run(debug=True, port=8000)


import pygame
from const import *

class HpBar:

    def __init__(self, screen):
        self.screen = screen
        self.height = HP_BAR_HEIGHT
        self.x = HP_BAR_X
        self.y = HP_BAR_Y

    def update(self, hp):
        size = 0
        if hp > 0:
            size = hp
        self.width = HP_BAR_WIDTH * size / PLAYER_HP;
        self.rect = pygame.Rect(self.x, self.y, self.width, self.height)
        self.circuit_rect = pygame.Rect(self.x, self.y,HP_BAR_WIDTH,self.height)

    def draw(self):
        pygame.draw.rect(self.screen, GREEN, self.rect)
        pygame.draw.rect(self.screen, WHITE, self.circuit_rect, 2)


from django.test import TestCase
from tests import factories

from marketplace.templatetags import breadcrumb

class BreadcrumbTest(TestCase):

    def setUp(self):
        self.cat1 = factories.CategoryFactory(name="cat1")
        self.cat2 = factories.CategoryFactory(name="cat2", parent=self.cat1)
        self.product = factories.ProductFactory(title="Brand name shampoo",
                                                primary_category=self.cat2)

    def check_breadcrumb(self, thing, expected):
        context = breadcrumb.generate_breadcrumb(thing)
        self.assertEqual(context, {
            "breadcrumbs": [("/", "Eco Market")] + expected,
        })

    def test_category(self):
        cat1_breadcrumb = [("/discover/cat1/", "cat1")]
        cat2_breadcrumb = cat1_breadcrumb + [("/discover/cat1/cat2/", "cat2")]
        # Empty URL is OK here as final crumb never shows URL
        product_breadcrumb = cat2_breadcrumb + [("", "Brand name shampoo")]
        self.check_breadcrumb(self.cat1, cat1_breadcrumb)
        self.check_breadcrumb(self.cat2, cat2_breadcrumb)
        self.check_breadcrumb(self.product, product_breadcrumb)


"""
loss functions for the models
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import torch
import torch.nn as nn

class CustomLoss(nn.Module):
  """
  A thin wrapper for loss functions
  L = sum_i (lam_i * loss_i)
  """
  def __init__(self, criterion_list, lam_list):
    super(CustomLoss, self).__init__()
    assert len(criterion_list) > 0
    # compile the list of loss functions into a module list
    # we assume same order of the loss w.r.t. the output vars
    assert len(criterion_list) == len(lam_list)
    self.criterion = nn.ModuleList(criterion_list)
    self.register_buffer('lam', torch.FloatTensor(lam_list))

  def forward(self, preds, targets):
    assert len(self.criterion) == len(preds)
    assert len(targets) <= len(self.criterion)
    # attach loss to each of the outputs
    # print(preds[0])
    # print(targets[0])
    loss = self.lam[0] * self.criterion[0](preds[0], targets[0])
    # print(loss)
    for idx in range(1, len(self.criterion)):
      target = targets[idx] if idx < len(targets) else None
      loss = loss + self.lam[idx] * self.criterion[idx](preds[idx], target)
    return loss

class SmoothedCrossEntropy(nn.Module):
  """Labled smoothing with cross entropy loss
  Note that this version is slightly different from our previous code
  Specifically, the smooth facotor is scaled by K / (K-1). This new
  implementation follows Hinton's orginal paper, and is slight faster and memory
  efficient (also supports ignore_index & weight)
  Weight: if specified, must be a pytorch tensor of size C
  """
  def __init__(self, label_smoothing=0.1, weight=None, ignore_index=None):
    super(SmoothedCrossEntropy, self).__init__()
    assert (label_smoothing < 1) and (label_smoothing > 0)
    self.label_smoothing = label_smoothing
    self.ignore_index = ignore_index
    self.register_buffer('weight', weight)

  def forward(self, pred, target):
    """ pred must be N * C """
    # create ignore mask
    if self.ignore_index is not None:
      ignore_mask = (target == self.ignore_index)
      valid_num_samples = float(pred.size(0) - sum(ignore_mask).item())
    else:
      valid_num_samples = float(pred.size(0))

    # divide the loss into two terms:
    # (\alpha - 1) \sum w_k y_k p_k - \alpha \sum w_k p_k /K
    log_prb = nn.functional.log_softmax(pred, dim=1)
    if self.weight is not None:
      log_prb = log_prb * self.weight
    ll_loss = log_prb.gather(dim=1, index=target.unsqueeze(1)).squeeze(1)
    smooth_loss = log_prb.mean(dim=1)
    losses = (self.label_smoothing - 1.0) * ll_loss \
             - self.label_smoothing * smooth_loss

    # mask out invalid samples & avg
    if self.ignore_index is not None:
      losses.masked_fill_(ignore_mask, 0.0)
    loss = losses.sum(0).div_(valid_num_samples)
    return loss

class KLDiv(nn.Module):
  """
    KL divergence for 3D attention maps
  """
  def __init__(self):
    super(KLDiv, self).__init__()
    self.register_buffer('norm_scalar', torch.tensor(1, dtype=torch.float32))

  def forward(self, pred, target=None):
    # get output shape
    batch_size, T = pred.shape[0], pred.shape[2]
    H, W = pred.shape[3], pred.shape[4]
    # N T HW
    atten_map = pred.view(batch_size, T, -1)
    log_atten_map = torch.log(atten_map)
    # print(torch.sum(target))
    # print(torch.max(target))
    # print(torch.min(target))
    if target is None:
      # uniform prior: this is really just neg entropy
      # we keep the loss scale the same here
      log_q = torch.log(self.norm_scalar / float(H*W))
      # \sum p logp - log(1/hw) -> N T
      kl_losses = (atten_map * log_atten_map).sum(dim=-1) - log_q
    else:
      log_q = torch.log(target.view(batch_size, T, -1))
      # \sum p logp - \sum p logq -> N T
      kl_losses = (atten_map * log_atten_map).sum(dim=-1) \
                  - (atten_map * log_q).sum(dim=-1)
    # N T -> N
    norm_scalar = T * torch.log(self.norm_scalar * H * W)
    kl_losses = kl_losses.sum(dim=-1) / norm_scalar
    kl_loss = kl_losses.mean()
    return kl_loss


"""
Инвест. проект заключается в том, чтобы научиться программировать
Регулярный доход:  500 долл. зарплата Junior C# разработчик
Регулярный расход: 150 долл. на зарплату за курсы + 390 долл. комуналка
Срок реализации проекта: 10 ЛЕТ
Ставка дисконтирования: 15 % уровень инфляции
"""
from itertools import accumulate
from functools import partial
import math

def calcNPV(cost0, income, cost, n, r):
    """Функция вычисляет NPV и DPP проекта"""

    NPVlist = [-cost0] + [(income - cost) / (1 + r) ** i for i in range(1, n + 1)]
    NPVaccum = list(accumulate(NPVlist))
    NPV = NPVaccum[-1]
    m = len([npvacc for npvacc in NPVaccum if npvacc < 0]) - 1
    DPP = m + abs(NPVaccum[m]) / NPVlist[m + 1] if len(NPVlist) > m + 1 else None
    return NPV, DPP

# входные данные
cost0 = 6000     # первоначальные затраты на учебу в ВШЭ
n = 10  # продолжительность проекта годы
income = 500*12*n  # Доходы от работы разработчиком в год
expensies = 490*12*n  # затраты на курсы программирования в месяц+комуналкa
r = 0.15

# считаем NPV, DPP
NPV, DPP = calcNPV(cost0, income, expensies, n, r)

# считаем IRR
calcNPV0 = partial(calcNPV, cost0=cost0, income=income, cost=expensies, n=n)
minNPV, IRR = math.inf, None
for r in range(1, 101):
    NPVirr = calcNPV0(r=r / 100)[0]
    if abs(NPVirr) < abs(minNPV):
        minNPV, IRR = NPVirr, r / 100
    print(r, NPVirr)

if IRR <= 0.01 or IRR >= 0.99:
    print("Скорее всего, IRR проекта с текущими параметрами отсутствует"
          " либо выбран слишком узкий диапазон изменения ставки дисконтирования")
    IRR = None

print("Учеба в ВЫШКЕ тот еще проект...")
print(f"ПРИВЕДЕННАЯ СТОИМОСТЬ = {NPV} USD, СРОК ОКУПАЕМОСТИ = {DPP} ЛЕТ, ДОПУСТИМЫЙ УРОВЕНЬ РЕАЛЬНОЙ ИНФЛЯЦИИ = {IRR*100} %: \n\tпроект {'' if NPV > 0 else 'не'}эффективен")
print("НО ЭТО НЕ ТОЧНО...")


import logging
import requests
from twilio.rest import Client

from .configs.config import TWILLO_FROM, TO_NUMBER, ACCOUNT_SID, AUTH_TOKEN

logger = logging.getLogger(__name__)

def send_twillo_sms_notification(content:str):
    """Sends a sms to default number with a provided message
    * Twillo account required - will use Twillo credit.
    * See [Twillo SMS](https://www.twilio.com/docs/sms) for more information.
    """
    logger.info('Sending text message')
    client = Client(ACCOUNT_SID, AUTH_TOKEN)
    message = client.messages \
                    .create(
                        body = content,
                        from_ = TWILLO_FROM,
                        to = TO_NUMBER
                    )
    logger.info(f"Message Status: {message.status}")

def send_textbelt_sms_notification(content:str):
    """Sends a sms to Env variable number with a provided message
    * Textbelt api offers 1 free text a day with key = 'textbelt'
    * See [Textbelt](https://textbelt.com/) for more details.
    """
    logger.info('Sending text message')
    resp = requests.post('https://textbelt.com/text',
        {
            'phone': TO_NUMBER,
            'message': content,
            'key': 'textbelt',
        }
    )
    logger.info(resp.json())


import json
from json import JSONDecodeError
from typing import Generator

import pytest

from mati import Client

VERIFICATION_RESP = {
    'expired': False,
    'identity': {'status': 'verified'},
    'steps': [],
    'flow': {
        'id': '5ae1c769ad10273b96fbc2b9',
        'name': 'Default flow'
    },
    'documents': [
        {
            'country': 'MX',
            'region': '',
            'photos': [
                'https://media.getmati.com/media/xxx',
                'https://media.getmati.com/media/yyy',
            ],
            'steps': [
                {'error': None, 'status': 200, 'id': 'template-matching'},
                {
                    'error': None,
                    'status': 200,
                    'id': 'mexican-curp-validation',
                    'data': {
                        'curp': 'CURP',
                        'fullName': 'LAST FIRST',
                        'birthDate': '01/01/1980',
                        'gender': 'HOMBRE',
                        'nationality': 'MEXICO',
                        'surname': 'LAST',
                        'secondSurname': '',
                        'name': 'FIRST',
                    },
                },
                {
                    'error': None,
                    'status': 200,
                    'id': 'document-reading',
                    'data': {
                        'fullName': {
                            'value': 'FIRST LAST',
                            'label': 'Name',
                            'sensitive': True,
                        },
                        'documentNumber': {
                            'value': '111',
                            'label': 'Document Number',
                        },
                        'dateOfBirth': {
                            'value': '1980-01-01',
                            'label': 'Day of Birth',
                            'format': 'date',
                        },
                        'expirationDate': {
                            'value': '2030-12-31',
                            'label': 'Date of Expiration',
                            'format': 'date',
                        },
                        'curp': {'value': 'CURP', 'label': 'CURP'},
                        'address': {
                            'value': 'Varsovia 36, 06600 CDMX',
                            'label': 'Address',
                        },
                        'emissionDate': {
                            'value': '2010-01-01',
                            'label': 'Emission Date',
                            'format': 'date',
                        },
                    },
                },
                {'error': None, 'status': 200, 'id': 'alteration-detection'},
                {'error': None, 'status': 200, 'id': 'watchlists'},
            ],
            'type': 'national-id',
            'fields': {
                'fullName': {
                    'value': 'FIRST LAST',
                    'label': 'Name',
                    'sensitive': True,
                },
                'documentNumber': {'value': '111', 'label': 'Document Number'},
                'dateOfBirth': {
                    'value': '1980-01-01',
                    'label': 'Day of Birth',
                    'format': 'date',
                },
                'expirationDate': {
                    'value': '2030-12-31',
                    'label': 'Date of Expiration',
                    'format': 'date',
                },
                'curp': {'value': 'CURP', 'label': 'CURP'},
                'address': {
                    'value': 'Varsovia 36, 06600 CDMX',
                    'label': 'Address',
                },
                'emissionDate': {
                    'value': '2010-01-01',
                    'label': 'Emission Date',
                    'format': 'date',
                },
            },
        }
    ],
    'hasProblem': False,
    'computed': {'age': {'data': 100}},
    'id': '5d9fb1f5bfbfac001a349bfb',
    'metadata': {'name': 'First Last', 'dob': '1980-01-01'},
}

def scrub_sensitive_info(response: dict) -> dict:
    response = scrub_access_token(response)
    response = swap_verification_body(response)
    return response

def scrub_access_token(response: dict) -> dict:
    try:
        resp = json.loads(response['body']['string'])
    except (JSONDecodeError, KeyError):
        pass
    else:
        if 'access_token' in resp:
            resp['access_token'] = 'ACCESS_TOKEN'
            try:
                user = resp['payload']['user']
            except KeyError:
                pass
            else:
                user['_id'] = 'ID'
                user['firstName'] = 'FIRST_NAME'
                user['lastName'] = 'LAST_NAME'
                resp['payload']['user'] = user
            response['body']['string'] = json.dumps(resp).encode('utf-8')
    return response

def swap_verification_body(response: dict) -> dict:
    if b'curp' not in response['body']['string']:
        return response
    response['body']['string'] = json.dumps(VERIFICATION_RESP).encode('utf-8')
    return response

@pytest.fixture(scope='module')
def vcr_config() -> dict:
    config = dict()
    config['filter_headers'] = [('Authorization', None)]
    config['before_record_response'] = scrub_sensitive_info  # type: ignore
    return config

@pytest.fixture
def client() -> Generator:
    yield Client('api_key', 'secret_key')

@pytest.fixture
def identity(client: Client) -> Generator:
    yield client.identities.create(
        client=client,
        nombres='Georg Wilhelm',
        primer_apellido='Friedrich',
        segundo_apellido='Hegel',
        dob='1770-08-27',
    )


"""
https://en.wikipedia.org/wiki/Strongly_connected_component
A directed graph is said to be strongly connected if every vertex is reachable from every other vertex.
A strongly connected component (SCC) of a directed graph is a maximal strongly connected subgraph.

O(V + E) to find all the strongly connected components
https://en.wikipedia.org/wiki/Kosaraju%27s_algorithm
https://www.geeksforgeeks.org/strongly-connected-components/
https://cp-algorithms.com/graph/strongly-connected-components.html

SPOJ CAPCITY
"""

from typing import List

def Kosaraju(g: List[List[int]]) -> List[int]:
    """
    Kosaraju's algorithm O(V + E)
    g: directed graph, graph[u] = [v where u -> v]
    return the number of scc and the partition of vertices
    """

    n = len(g)
    order, visited = [], [False] * n
    for u in range(n):
        if visited[u]: continue
        visited[u] = True
        stack = [(u, 0)]
        while stack:  # use stack instead of dfs to avoid stack overflow from deep recursion
            cur, iter = stack.pop()
            if iter < len(g[cur]):
                stack.append((cur, iter + 1))
                v = g[cur][iter]
                if not visited[v]:
                    visited[v] = True
                    stack.append((v, 0))
            else:
                order.append(cur)
    gt = [[] for _ in range(n)]  # transpose of graph
    for u in range(n):
        for v in g[u]:
            gt[v].append(u)
    scc, cnt = [-1] * n, 0
    for u in reversed(order):
        if scc[u] >= 0: continue
        scc[u] = cnt
        q = [u]
        while q:
            for v in gt[q.pop()]:
                if scc[v] < 0:
                    scc[v] = cnt
                    q.append(v)
        cnt += 1
    return cnt, scc

'''
https://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm
'''

# To implement


import json
from Chain import Chain
from WalletMainService import WalletMainService
from TxGetter import TxGetterScan

if __name__ == '__main__':
    wallet_main_service_scan = WalletMainService(TxGetterScan())
    chains = open('ChainData.json')
    chains_data = json.load(chains)
    to_analyse = open('ToAnalyse.json')
    to_analyse_data = json.load(to_analyse)

    for wallet in to_analyse_data['wallets']:
        needed_chain_data = next((x for x in chains_data['chains'] if x['chain'] == wallet['chain']), None)
        wallet_add = wallet['addr']

        if needed_chain_data is None:
            raise Exception("Chain " + wallet['chain'] + " not found in ChainData.json")

        chain_obj = Chain(needed_chain_data['chain'], needed_chain_data['api_base_url'], needed_chain_data['currency'],
                          needed_chain_data['decimals'], needed_chain_data['api_key'])

        wallet_main_service_scan.formatForWallet(wallet_add, chain_obj)


import requests
from app import conf

ID_KEY = conf.getSetting('VK', 'id_key')
SECURE_KEY = conf.getSetting('VK', 'secure_key')
SERVICE_KEY = conf.getSetting('VK', 'service_key')
VERSION = conf.getSetting('VK', 'version')

def refactDict(func):
    # Декоратор для измения списка словарей друзей
    def wrapper(accesToken):
        friends = func(accesToken)
        for friend in friends:
            if 'sex' in friend:
                if friend['sex'] == 0:
                    friend['sex'] = 'не указан'
                elif friend['sex'] == 1:
                    friend['sex'] = 'женский'
                elif friend['sex'] == 2:
                    friend['sex'] = 'мужской'
            if 'online' in friend:
                if friend['online'] == 0:
                    friend['online'] = 'оффлайн'
                elif friend['online'] == 1:
                    friend['online'] = 'онлайн'
            if 'country' not in friend:
                friend['country'] = {'title': 'не указана'}
            if 'city' not in friend:
                friend['city'] = {'title': 'не указан'}
        return friends

    return wrapper

@refactDict
def getFriendsByToken(access_token):
    # Получение друзей пользователя с использованием токена
    friends = []
    fields = ['country', 'city', 'sex', 'photo_200_orig', ]
    responce = requests.get('https://api.vk.com/method/friends.get',
                            params={'access_token': access_token, 'order': 'random', "v": VERSION,
                                    'fields': ', '.join(fields), 'count': 5})
    if 'error' not in responce:
        friends = responce.json()['response']['items']
    return friends

@refactDict
def getFriendsById(idUser):
    # Получение списка друзей по его vk ID
    friends = []
    fields = ['country', 'city', 'sex', 'photo_200_orig', ]
    responce = requests.get('https://api.vk.com/method/friends.get',
                            params={'access_token': SERVICE_KEY, 'user_id': idUser, 'order': 'random', "v": VERSION,
                                    'count': 5})
    responce = responce.json()
    if 'error' not in responce:
        idFriends = responce['response']['items']
        friends = list(map(getUserDataById, idFriends))
    print(friends)
    return friends

def getUserDataByToken(access_token):
    # Получение имени и фамилии пользователя по токену
    responce = requests.get('https://api.vk.com/method/users.get',
                            params={'access_token': access_token, "v": VERSION, 'name_case': 'gen'})
    responce = responce.json()
    userName = responce['response'][0]
    return userName

def getUserDataById(idUser):
    # Получение имени и фамилии пользователя по vk id
    fields = ['country', 'city', 'sex', 'photo_200_orig', 'online']
    responce = requests.get('https://api.vk.com/method/users.get',
                            params={'user_id': idUser, 'access_token': SERVICE_KEY, "v": VERSION, 'name_case': 'gen',
                                    'fields': ', '.join(fields)})
    responce = responce.json()
    userName = responce['response'][0]
    return userName

def getAccessKey(code):
    responce = requests.get('https://oauth.vk.com/access_token',
                            params={'client_id': ID_KEY, 'client_secret': SECURE_KEY,
                                    'redirect_uri': 'https://flaskvkapitest.herokuapp.com', 'code': code})
    if responce.status_code == 200:
        data = responce.json()
        try:
            access_token = data['access_token']
        except KeyError:
            access_token = None
    return access_token


import json
import subprocess
import pathlib
import pytest
import sys

from openassetio.errors import BatchElementError
from openassetio.hostApi import ManagerFactory

# pylint: disable=no-self-use
# pylint: disable=invalid-name,redefined-outer-name
# pylint: disable=missing-class-docstring,missing-function-docstring

class Test_simpleResolver_errors:
    def test_when_lib_initialization_fails_then_error_written_to_stderr_and_return_code_non_zero(
        self, monkeypatch
    ):
        monkeypatch.delenv("OPENASSETIO_DEFAULT_CONFIG", raising=False)
        result = execute_cli()
        expected_message = [
            "ERROR: No default manager configured, "
            f"check ${ManagerFactory.kDefaultManagerConfigEnvVarName}"
        ]
        assert result.stderr.splitlines() == expected_message
        assert result.returncode == 1

    def test_when_env_set_and_no_args_then_usage_printed_and_return_code_is_one(
        self, test_config_env  # pylint: disable=unused-argument
    ):
        result = execute_cli()
        expected_message = [
            "usage: simpleResolver.py [-h] traitset entityref",
            "simpleResolver.py: error: the following arguments are required: traitset, entityref",
        ]
        assert result.stderr.splitlines() == expected_message
        assert result.returncode == 2

    def test_when_entity_ref_and_trait_set_valid_then_expected_data_output_and_return_code_zero(
        self, test_config_env  # pylint: disable=unused-argument
    ):
        result = execute_cli("animal,named", "bal:///cat")
        data = json.loads(result.stdout)
        expected_data = {
            "animal": {"species": "🐈", "age": 12},
            "named": {"name": "Martin"},
        }
        assert data == expected_data
        assert result.returncode == 0

    def test_when_entity_ref_valid_and_trait_set_invalid_then_data_empty_and_return_code_zero(
        self, test_config_env  # pylint: disable=unused-argument
    ):
        result = execute_cli("someMissingTrait", "bal:///cat")
        data = json.loads(result.stdout)
        assert data == {}
        assert result.returncode == 0

    def test_when_entity_ref_invalid_then_error_and_return_code_wrapped(
        self, test_config_env  # pylint: disable=unused-argument
    ):
        result = execute_cli("named", "bal:///doesNotExist")
        # Don't test the specific message as it couples to BAL specifics
        assert result.stderr.startswith("ERROR:")
        assert result.returncode == int(BatchElementError.ErrorCode.kEntityResolutionError)

def execute_cli(*args):
    """
    Executes simplerResolver with the supplied args, as per subprocess.run.
    """
    this_file = pathlib.Path(__file__)
    cli_path = this_file.parent / "simpleResolver.py"
    all_args = [sys.executable, str(cli_path)]
    all_args.extend(args)
    # We explicitly don't want an exception to be raised.
    # pylint: disable=subprocess-run-check
    return subprocess.run(all_args, capture_output=True, encoding="utf-8")

@pytest.fixture
def test_config_env(monkeypatch, tmpdir):
    """
    A fixture that configures the process environment such that the
    OpenAssetIO default config points to the example toml file.
    """
    this_file = pathlib.Path(__file__)
    config_file = this_file.parent / "bal_animals_openassetio_config.toml"
    monkeypatch.setenv("OPENASSETIO_DEFAULT_CONFIG", str(config_file))
    # Deliberately not same directory as the library JSON file, so
    # relies on ${config_dir} interpolation.
    monkeypatch.chdir(tmpdir)