from aiogram import Router
from aiogram.filters import Command, CommandStart
from aiogram.fsm.context import FSMContext
from aiogram.types import Message
from aiogram.utils.markdown import hbold

from hpc_bot.exceptions import ExistChatError
from hpc_bot.telegram.fsm import UserMode
from hpc_bot.telegram.manager import ModelManager
from hpc_bot.telegram.messages import (
    CURRENT_STATE_TEXT,
    EXIST_CHAT_TEXT,
    HELLO_TEXT,
    HELP_TEXT,
    INLINE_MODE_TEXT,
    RESET_CHAT_TEXT,
    WINDOW_MODE_TEXT,
)

command_router: Router = Router(name='command_router')

@command_router.message(CommandStart())
async def command_start_handler(message: Message, manager: ModelManager, state: FSMContext) -> None:
    """
    Начать взаимодействие с ботом `/start` командой
    :param message: сообщение от пользователя
    :param manager: менеджер управления инференсом модели.
    :param state: состояние конечного автомата пользователя.
    :return:
    """
    await state.set_state(UserMode.window)

    try:
        await manager.start_conversion(message.chat.id, message.from_user.username)
    except ExistChatError:
        await message.answer(EXIST_CHAT_TEXT)
    else:
        await message.answer(HELLO_TEXT.format(username=hbold(message.from_user.full_name)))

@command_router.message(Command('reset_context'))
async def command_clean_context(message: Message, manager: ModelManager) -> None:
    """
    Очистить весь контекст переписки с ботом
    :param message: сообщение от пользователя
    :param manager: менеджер управления инференсом модели.
    :return:
    """
    await manager.reset_context(message.chat.id)
    await message.answer(RESET_CHAT_TEXT)

@command_router.message(Command('help'))
async def command_help_handler(message: Message) -> None:
    """
    Вернуть подсказку для общения с ботом
    :param message: сообщение от пользователя
    :return:
    """
    await message.answer(HELP_TEXT)

@command_router.message(Command('inline'))
async def command_inline_mode(message: Message, state: FSMContext) -> None:
    """
    Команда на включение режима линейной переписки с пользователем.
    :param message: сообщение от пользователя.
    :param state: состояние конечного автомата пользователя.
    :return:
    """
    await state.set_state(UserMode.inline)
    await message.answer(INLINE_MODE_TEXT)

@command_router.message(Command('window'))
async def command_window_mode(message: Message, state: FSMContext) -> None:
    """
    Комманда на включение режима скользящего окна переписки с пользователем.
    :param message: сообщение от пользователя.
    :param state: состояние конечного автомата пользователя.
    :return:
    """
    await state.set_state(UserMode.window)
    await message.answer(WINDOW_MODE_TEXT)

@command_router.message(Command('current_mode'))
async def command_current_mode(message: Message, state: FSMContext) -> None:
    """
    Команда на получение текущего режима переписки с пользователем.
    :param message: сообщение от пользователя.
    :param state: состояние конечного автомата пользователя.
    :return:
    """
    state = await state.get_state()
    await message.answer(CURRENT_STATE_TEXT.format(state=state))


from dagster import (
    AssetKey,
    DefaultSensorStatus,
    EventLogEntry,
    RunRequest,
    SensorEvaluationContext,
    asset_sensor,
)

from medallion_weather.jobs import all_assets_job
from medallion_weather.resources.configs import define_all_assets_job_run_config

@asset_sensor(
    asset_key=AssetKey(["GCP", "test_pub_dataset", "gold_aqi_with_pm_asset"]),
    job=all_assets_job,
    default_status=DefaultSensorStatus.STOPPED,
    minimum_interval_seconds=30,
    name="sensor_of_gold_aqi_with_pm_asset",
    description="This is an asset sensor with respect to `gold_aqi_with_pm_asset` from the domain `medallion_air`.",
)
def from_air_gold_aqi_with_pm_asset(_context: SensorEvaluationContext, asset_event: EventLogEntry):
    run_key = _context.cursor
    run_config = define_all_assets_job_run_config()
    run_req: RunRequest = all_assets_job.run_request_for_partition(
        run_key=run_key,
        partition_key=asset_event.dagster_event.partition,
        run_config=run_config,
    )
    _context.update_cursor(run_key)
    yield run_req


"""
一つのフォルダにある複数Excelブックの調書の施設名とか高さを一つのExcelに一覧にしたいときに使うが、微妙微妙微妙…
pandasを使ったほうが、きれいになるかと思ったが…
range_copy_from_multipul_sheets.pyを使ったほうが無難
"""

import os
import pandas
import xlwt
import xlrd
import itertools
import glob
from tqdm import tqdm

dir = "D:/三八DB/02収集資料/区域調書_移動不可/*.xls*"  # 集計したいExcelが入っているフォルダを指定

row = 0
wb = xlwt.Workbook()
add_sheet = wb.add_sheet("new", cell_overwrite_ok=True)
xls_format = "D:/三八DB/02収集資料/高さ4.xls"  # 保存先Excelを指定
 # シートを指定
"""
range_copy関数について
start_row1,end_row1,col_name1,start_row2,end_row2,col_name2
の設定はいったんExcelシートをpandasで読み込んでデータフレームに変換した段階で、確認しながら入れたほうが良い。
df=pandas.read_excel(FilePath,sheet_name)←ここで
Excelの行列番号とは異なるので注意。
start_row1は切り出し始めの行番号
end_row1は切り出し終わりの行番号
col_name1はデータフレーム上での列名(列名がない場合は'Unnamed: 5'とかになる)

start_row2は範囲その2のこと要らなかったらコメントアウト
"""
def range_copy(sheet_name,start_row1,end_row1,col_name1,kasho_num_row="",kasho_num_col=""):
    try:  # ブックが読み込めてシートがあるかどうかtry
        df = pandas.read_excel(FilePath, sheet_name)

        #write_to_xls(All_list)
    except:
        xlrd.biffh.XLRDError  # シートがなくてエラーになった場合の処理
        print(FilePath + "に{}はありませんでした".format(sheet_name))
        # write_to_xls(FilePath + "に指定のシートはありませんでした")
    # except xlrd.biffh.XLRDError:  # xlrdだとシート保護されたExcelは読み込めない！
    #     print(FilePath + "はシート保護され取得できませんでした")
        All_list = []
        #write_to_xls(All_list)

    else:  # 無事読み込めたときの処理
        All_list = []
        # 箇所番号
        if kasho_num_row != "" and kasho_num_col != "":
            num = df.loc[kasho_num_row, kasho_num_col]
            All_list.append(num)
        # 範囲その1
        range_copy1 = df.loc[
            start_row1:end_row1, [col_name1]
        ]  # .loc(行番号,["列名"]でdataframeから切り出せる
        range_copy1_l = range_copy1.values.tolist()  # dataframeをnumpy配列にする
        range_copy1 = list(
            itertools.chain.from_iterable(range_copy1_l)
        )  # 2次元のリストを平坦化する

        # 範囲その2　要らない場合はコメントアウト
        # range_copy2 = df.loc[
        #     start_row2:end_row2, [col_name2]
        # ]  # .loc(行番号,["列名"]でdataframeから切り出せる
        # range_copy2_l = range_copy2.values.tolist()  # dataframeをnumpy配列にする
        # range_copy2 = list(
        #     itertools.chain.from_iterable(range_copy2_l)
        # )  # 2次元のリストを平坦化する

        All_list.extend(range_copy1)
        # All_list.extend(range_copy2)  # 範囲その1のリストに範囲その2のリストを追加する　要らない場合はコメントアウト
        # write_to_xls(All_list)
    return All_list

def write_to_xls(All_list):
    global row
    global i
    for value in All_list:  # value(リスト)をExcelに書き出していく(1リスト1行、1要素1セルで)
        add_sheet.write(row, i, str(value))
        i += 1

for FilePath in tqdm(glob.glob(dir)[65:]):
    # for val in range (range_copy1.size):
    #     print(range_copy1.values[val,0])
    #
    # for val in range (range_copy2.size):
    #     print(range_copy2.values[val,0])
    i=0
    row+=1
    try:
        FilePath in (dir)[65:]
    except AssertionError:  # ファイルが壊れている場合の処理
        print(FilePath + "はファイルが破損している可能性があります")
        All_list = [FilePath + "はファイルが破損している可能性があります"]
        add_sheet.write(row,i,str(All_list))
        row += 1
        #write_to_xls(All_list)
    else:
        write_to_xls(range_copy(
        sheet_name="様式4-3",
        kasho_num_row=1,
        kasho_num_col="Unnamed: 7",
        start_row1=7,
        end_row1=21,
        col_name1="Unnamed: 2",
        # start_row2=24,
        # end_row2=26,
        # col_name2="Unnamed: 21",
    ))
    write_to_xls(range_copy(
        sheet_name="様式4-3 (1)",
        kasho_num_row=1,
        kasho_num_col="Unnamed: 7",
        start_row1=7,
        end_row1=21,
        col_name1="Unnamed: 2",
        # start_row2=24,
        # end_row2=26,
        # col_name2="Unnamed: 21",
    ))
    write_to_xls(range_copy(
        sheet_name="様式4-3 (2)",
        start_row1=7,
        end_row1=21,
        col_name1="Unnamed: 2",
        # start_row2=24,
        # end_row2=26,
        # col_name2="Unnamed: 21",
    ))

    wb.save(xls_format)


from redis import Redis

if __name__ == '__main__':
    try:
        redis_client = Redis(host="192.168.84.128", port=6379, decode_responses=True)
        result = redis_client.get("name5")
        print(result, type(result))
        # str1 = result.decode("utf-8")
        # print(str1, type(str1))
    except Exception as e:
        print(e)


#!/bin/env python3
"""Convert giellalt dict to apertium bidix."""

import sys
import xml.etree.ElementTree
from argparse import ArgumentParser, FileType
from typing import TextIO

def giella2apes(giella: str) -> str:
    """Convert pos tag from giellalt dict to apertium."""
    if giella == "Adv":
        return "adv"
    elif giella == "A":
        return "adj"
    elif giella == "CC":
        return "cnjcoo"
    elif giella == "CS":
        return "cnjsub"
    elif giella == "Interj":
        return "ij"
    elif giella == "N":
        return "n"
    elif giella == "Num":
        return "num"
    elif giella == "Pcle":
        return "pcle"
    elif giella == "Po":
        return "post"
    elif giella == "Pr":
        return "pr"
    elif giella == "Pron":
        return "prn"
    elif giella == "Det":
        return "det"
    elif giella == "Prop":
        return "np"
    elif giella == "V":
        return "vblex"
    else:
        print(f"missing giella tag conversion for {giella}!")
        return "x"
    return "x"

def handle_mg(mg: xml.etree.ElementTree.Element, apes: dict,
              translations: list, transposes: list, examples: list,
              tranxamples: list):
    """Convert mg."""
    for morph in mg:
        if morph.tag == "tg":
            for trans in morph:
                if trans.tag == "t":
                    translations.append(trans.text)
                    transposes.append(giella2apes(trans.attrib["pos"]))
                elif trans.tag == "xg":
                    for example in trans:
                        if example.tag == "x":
                            examples.append(example.text)
                        elif example.tag == "xt":
                            tranxamples.append(example.text)
                        else:
                            print(f"Unrecognised {example.tag} under {trans.tag}")
                elif trans.tag == "re":
                    print(f"  context restriction? {trans.text}")
                else:
                    print(f"Unrecognised {trans.tag} under {morph.tag}")
        else:
            print(f"Unrecognised {morph.tag} under {mg.tag}")

def handle_e(e: xml.etree.ElementTree.Element, apes: dict, output: TextIO):
    """Convert and print dictionary entry e."""
    lemma = None
    pos = None
    translations = []
    transposes = []
    examples = []
    tranxamples = []
    for child in e:
        if child.tag == "lg":
            for lex in child:
                lemma = lex.text
                pos = giella2apes(lex.attrib["pos"])
        elif child.tag == "mg":
            handle_mg(child, apes, translations, transposes,
                      examples, tranxamples)
        else:
            print(f"Unrecognised {child.tag} under {e.tag}")
            sys.exit(1)
    if not lemma:
        print("Couldn't find source lemma in this <e>:", e)
        return
    if not pos:
        print("Couldnt't  find source pos in this <e>:", e)
        return
    src_in_bidix = False
    if lemma + "\t" + pos in apes:
        src_in_bidix = True
        print(f"    {lemma}.{pos} (already in dix)")
    else:
        print(f"    {lemma}.{pos}")
    for i, trans in enumerate(translations):
        print(f"        {trans}.{transposes[i]}")
        default = "y"
        if " " in trans:
            print("      (lemmas with spaces get extra weight by default)")
            default = "3"
        elif "xxx" in trans:
            print("     (lemmas with todo symbols get ignored")
            default = "n"
        if src_in_bidix:
            if apes[lemma + "\t" + pos] == trans + "\t" + transposes[i]:
                print("(already in bidix skipping...)")
                continue
            else:
                print("     (existing in bidix as: " +
                      apes[lemma + "\t" + pos] + "; adding default weight)")
                default = "2"
        if i in examples:
            print(f"\t\"{examples[i]}\" ~ \"{tranxamples[i]}\"")
        answer = input("yes, no, quit, or integer weight? default: " +
                       default + ". ")
        if answer == "":
            answer = default
        if answer in ["y", "yes"]:
            print(f"    <e>
<l>{lemma}<s n=\"{pos}\"/></l>"
                  f"<r>{trans}<s n=\"{transposes[i]}\"/></r>
</e>",
                  file=output)
        elif answer in ["n", "no"]:
            pass
        elif answer in ["q", "quit"]:
            sys.exit(0)
        elif answer.isdigit():
            print(f"    <e w=\"{answer}\">
<l>{lemma}<s n=\"{pos}\"/></l>"
                  f"<r>{trans}<s n=\"{transposes[i]}\"/></r>
</e>",
                  file=output)
        else:
            print(f"Wrong answer {answer}! Skippings to next...")

def read_apes(apefile: TextIO):
    """Extract bidix data from apertium dictionary."""
    apes = read_xml(apefile)
    root = apes.getroot()
    bidix = {}
    for child in root:
        if child.tag == "section":
            for entry in child:
                if entry.tag == "e":
                    srclemma = None
                    trglemma = None
                    srcpos = None
                    trgpos = None
                    for part in entry:
                        if part.tag == "i":
                            srclemma = part.text
                            trglemma = part.text
                            for tags in part:
                                if tags.tag == "s":
                                    srcpos = tags.attrib["n"]
                                    trgpos = tags.attrib["n"]
                                    break
                        elif part.tag == "p":
                            for side in part:
                                if side.tag == "l":
                                    srclemma = side.text
                                    for tags in side:
                                        if tags.tag == "s":
                                            srcpos = tags.attrib["n"]
                                            break
                                elif side.tag == "r":
                                    trglemma = side.text
                                    for tags in side:
                                        if tags.tag == "s":
                                            trgpos = tags.attrib["n"]
                                            break
                        elif part.tag == "re":
                            srclemma = "__regexp__"
                            trglemma = "__regexp__"
                    if not srclemma:
                        print("missing source lemma in: ", entry)
                        continue
                    if not srcpos:
                        print("missing source pos in: ", entry)
                        continue
                    if not trglemma:
                        print("missing target lemma in: ", entry)
                        continue
                    if not trgpos:
                        print("missing target pos in: ", entry)
                        continue
                    bidix[srclemma + "\t" + srcpos] = trglemma + "\t" + trgpos
    return bidix

def main():
    """Command-line interface to convertion."""
    arp = ArgumentParser(description="Merge giellalt dics with apertiums")
    arp.add_argument("-i", "--input", metavar="INFILE", type=open,
                     dest="infile", help="read giellalt dict from INFILE",
                     required=True)
    arp.add_argument("-o", "--output", metavar="OUTFILE", type=FileType("w"),
                     dest="outfile", help="write bidix fragment to OUTFILE",
                     required=True)
    arp.add_argument("-r", "--reference", metavar="APEFILE", type=open,
                     dest="apefile", help="read reference bidix from APEFILE")
    arp.add_argument("-y", "--accept-default", action="store_true",
                     help="non-interactive mode that selects default always")
    arp.add_argument("-v", "--verbose", action="store_true",
                     help="print verbosely while opeartioning")
    options = arp.parse_args()
    if options.verbose:
        print(f"Reading apertium bidix from {options.apefile.name}...")
    apes = read_apes(options.apefile)
    if options.verbose:
        print(f"Reading giellaLT dictionary from {options.infile.name}...")
    giellalt = read_xml(options.infile)
    root = giellalt.getroot()
    for child in root:
        if child.tag == "lics":
            print("", file=options.outfile)
        elif child.tag == "e":
            handle_e(child, apes, options.outfile)

def read_xml(xmlfile: TextIO):
    """Read a GiellaLT dictionary file."""
    xmldict = xml.etree.ElementTree.parse(xmlfile)
    return xmldict

if __name__ == "__main__":
    main()


import torch
import torch.onnx
from collections import OrderedDict
import argparse
from mmcv import Config, DictAction
from mmpose.models import build_posenet

def parse_args():
    parser = argparse.ArgumentParser(description='Train a pose model')
    parser.add_argument('--config',default="configs/top_down/deeppose/coco/npu_deeppose_res50_coco_256x192.py",help='train config file path')
    args = parser.parse_args()
    return args

def convert():
    args = parse_args()
    cfg = Config.fromfile(args.config)
    model = build_posenet(cfg.model)
    model.eval()
    print(model)
    input_names = ["actual_input_1"]
    output_names = ["output1"]
    x = torch.randn(64, 3, 256, 192)
    torch.onnx.export(model.backbone, x, "deeppose_npu.onnx", input_names=input_names, output_names=output_names,
                      opset_version=11)

if __name__ == "__main__":
    convert()


"""
Unit and regression test for the pyIsoP package.

"""
# Uses the pytest fixtures decorator to take the energy grid and use it for a series of tests including
# values, writing, isotherms and etc
# Import package, test suite, and other packages as needed
import pyIsoP
import pytest
import sys

#%%
def test_pyIsoP_imported():
        """Sample test, will always pass so long as import statement worked"""
        assert "pyIsoP" in sys.modules
#%%
@pytest.fixture
def compute_grid():
        import os
        from pyIsoP import grid3D, forcefields, writer
        path_to_file = os.path.dirname(pyIsoP.__file__)+'/data/ZIF-4_mod.cif'
        t1=grid3D.grid3D(path_to_file,spacing=0.5 )
        f1=forcefields.forcefields(t1,sigma=3.95, epsilon=46,forcefield=os.path.dirname(pyIsoP.__file__)+'/../forcefield/UFF')
        t2= grid3D.grid3D.grid_calc(t1,"lj",f1)
        return t2
#%%
@pytest.fixture
def compute_grid_dask():
        import os
        from pyIsoP import grid3D, forcefields, writer
        path_to_file = os.path.dirname(pyIsoP.__file__)+'/data/ZIF-4_mod.cif'
        t1=grid3D.grid3D(path_to_file,spacing=0.5 )
        f1=forcefields.forcefields(t1,sigma=3.95, epsilon=46,forcefield=os.path.dirname(pyIsoP.__file__)+'/../forcefield/UFF')
        grid_dask= grid3D.grid3D.grid_calc_dask(t1,f1)
        t1.pot=grid_dask.compute()
        return t1
#%%
@pytest.fixture
def compute_ml():
        import os
        import pyIsoP
        from pyIsoP import machlearn
        print("Testing the machine learning using GPR")
        path_to_file = os.path.dirname(pyIsoP.__file__)+'/data/for_gpr_large.dat'
        ml=machlearn.machlearn(restarts=1)
        ml=machlearn.machlearn.GPR4n1(ml,path_to_file,0.1)
        return ml
#%%
@pytest.fixture
def compute_histo(compute_grid):
        from pyIsoP import histo
        h=histo.histo()
        h=histo.histo.grid2histo(compute_grid,h)
        return h
#%%
def test_grid_values(compute_grid):
        import numpy as np
        print("Testing the energy grid calculation")
        print(str(np.min(compute_grid.pot))), "Energy minimum looks good!"
        assert np.abs(np.min(np.round(compute_grid.pot, decimals=2)+1819.74)<=1E-4), "Grid minimum does not match reference"
#     assert(np.max(compute_grid.pot_repeat)==)
#%%

def test_grid_values_dask(compute_grid_dask):
        test_grid_values(compute_grid_dask)

#%%
def test_write_grid(compute_grid):
        import pyIsoP
        import os
        from pyIsoP import writer
        import numpy as np

        print("Testing the grid writer")
        path_to_out_vtk = os.path.dirname(pyIsoP.__file__)+'/data/zif-4_grid'
        path_to_out_pdb = os.path.dirname(pyIsoP.__file__)+'/data/zif-4_replicated.pdb'
        print("Writing .vts and .pdb tests into the data folder")
        writer.writer.write_vts(compute_grid,path_to_out_vtk, 1,1,1)
        writer.writer.write_frame(compute_grid,path_to_out_pdb)
        #should we assert something..?
#%%
def test_write_grid_dask(compute_grid_dask):
        test_write_grid(compute_grid_dask)

#%%
def test_histo_vals(compute_histo):
        import numpy as np
        print("Testing histogram values")
        # some check on the zif-4 energy histogram, within 1 kBT of expected
        reference_hist = np.array([-23.89147811, -23.40855962, -22.92564113, -22.44272265,
           -21.95980416, -21.47688567, -20.99396718, -20.5110487 ,
           -20.02813021, -19.54521172, -19.06229324, -18.57937475,
           -18.09645626, -17.61353777, -17.13061929, -16.6477008 ,
           -16.16478231, -15.68186382, -15.19894534, -14.71602685,
           -14.23310836, -13.75018988, -13.26727139, -12.7843529 ,
           -12.30143441, -11.81851593, -11.33559744, -10.85267895,
           -10.36976046,  -9.88684198,  -9.40392349,  -8.921005  ,
            -8.43808651,  -7.95516803,  -7.47224954,  -6.98933105,
            -6.50641257,  -6.02349408,  -5.54057559,  -5.0576571 ,
            -4.57473862,  -4.09182013,  -3.60890164,  -3.12598315,
            -2.64306467,  -2.16014618,  -1.67722769,  -1.19430921,
            -0.71139072,  -0.22847223])
        print(str(np.sum(np.abs(compute_histo.E-reference_hist))))
        assert np.sum(np.abs(compute_histo.E-reference_hist))<=0.1,"Histogram doesnot matches reference!"
#%%
def test_machlearn(compute_ml):
        print("Testing the trained ML model")
        import numpy as np
        print(str(compute_ml.gp.predict([[6,0.5,10,5]])))
        assert np.abs(compute_ml.gp.predict([[6,0.5,10,5]])-3) <=1, "Coordination number predicted doesn't agree with reference!"  # See if the predicted n1 is within a certain expected range
#%%
def test_machlearn2():

        print("Testing the trained ML model")
        import numpy as np
        import joblib
        import pyIsoP
        import os
        path_to_joblib_dump=os.path.dirname(pyIsoP.__file__)+'/data/gprmodel_pyIsoP.joblib'
        gpr = joblib.load(path_to_joblib_dump)
#        print(str(gpr.predict([[6,0.5,10,5]])))
        assert np.abs(gpr.predict([[6,0.5,10,5]])-3) <=1, "Coordination number predicted doesn't agree with reference!"  # See if the predicted n1 is within a certain expected range

#%%
def test_predictor(compute_histo, compute_ml):
        from pyIsoP import predictor
        import numpy as np
        print("Testing the isotherm prediction routine")

        n_pressures=20
        P=np.linspace(1E-5,100,n_pressures)
        T=298
        Vf=0.66
        lcd=5.1
        pld=2.4
        X_test           = np.array([[Vf, lcd, pld],]*n_pressures)
        X_test           = np.hstack((np.reshape(np.log10(P*1E5),(n_pressures,1)), X_test))
        n1=compute_ml.gp.predict(X_test)
        refvals=np.array([[ 23.78752723],
           [ 43.53239219],
           [ 44.43192367],
           [ 44.89588641],
           [ 45.1897038 ],
           [ 45.39587854],
           [ 45.5499096 ],
           [ 45.66999489],
           [ 45.76656782],
           [ 45.84609705],
           [ 45.91283278],
           [ 45.96969572],
           [ 46.01876613],
           [ 46.06156964],
           [ 46.09925272],
           [ 46.1326949 ],
           [ 46.16258297],
           [ 46.18946154],
           [ 46.21376834],
           [ 46.23585941]])
#        h=compute_histo

        g_L_CH2=predictor.predictors.predict_isotherm(T,P,Vf,compute_histo,n1,epsilon=46,MA=14)
        print(g_L_CH2)
        print(refvals)
        print(str(np.sum(np.abs(g_L_CH2-refvals))))
        assert(np.sum(np.abs(g_L_CH2-refvals))<=5), "The predicted isotherm does not match the reference!"
#%%


from LOTUS_regression.regression import mzm_regression
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import datetime
from LOTUS_regression.predictors.seasonal import add_seasonal_components

''' Code to apply MLR, ILT or new predictors to the Uccle data
    For data/model montly mean plots you need to add seasonal terms in the model'''

# boolean to run the code w.r.t. to reltropop of the absolute altitude
reltropop = False

def plotmlr_perkm(pX, pY, pRegOutput, pltitle, plname):
    plt.close('all')

    fig, ax = plt.subplots()
    # plt.title('Total O3')
    plt.xlabel('Years')
    # plt.ylabel('Residuals')
    # plt.ylabel('Thickness of the ozone layer [DU]')
    plt.ylabel('Total O3 Relative Monthly Means')
    # plt.title(pltitle)

    # plt.plot(pX, pY, label='Residuals', color='blue')
    plt.plot(pX, pY, label='Data', color='blue', linewidth = 1)

    plt.plot(pX, pRegOutput, label='Model', color='red', linewidth = 1.2)

    ax.legend(loc='upper right', frameon=True, fontsize='small')

    # plt.savefig('/home/poyraden/Analysis/MLR_Uccle/Plots/TotalO3/' + plname + '.pdf')
    # plt.savefig('/home/poyraden/Analysis/MLR_Uccle/Plots/TotalO3/' + plname + '.eps')
    # plt.savefig('/home/poyraden/Analysis/MLR_Uccle/Plots/Uccle_50years/DataModel/' + plname + '.pdf')
    # plt.savefig('/home/poyraden/Analysis/MLR_Uccle/Plots/Uccle_50years/DataModel/' + plname + '.eps')
    plt.show()
    plt.close()
    plt.close()

###

def plotresidual(pX, pRegOutput, pltitle, plname):
    plt.close('all')

    fig, ax = plt.subplots()
    plt.title('Total O3 Relative Monthly Anamolies')
    # plt.title('Total O3 Monthly Means')

    plt.xlabel('Years')
    # plt.ylabel('Residuals')
    plt.ylabel('Residuals (Data - Model)')

    # plt.plot(pX, pY, label='Residuals', color='blue')
    # plt.plot(pX, pY, label='Data', color='blue')

    plt.plot(pX, pRegOutput, label='Model', color='black', linewidth = 0.75)

    # ax.legend(loc='upper right', frameon=True, fontsize='small')

    # plt.savefig('/home/poyraden/Analysis/MLR_Uccle/Plots/TotalO3/' + plname + '.pdf')
    # plt.savefig('/home/poyraden/Analysis/MLR_Uccle/Plots/TotalO3/' + plname + '.eps')
    # plt.savefig('/home/poyraden/Analysis/MLR_Uccle/Plots/Uccle_50years/DataModel/' + plname + '.pdf')
    # plt.savefig('/home/poyraden/Analysis/MLR_Uccle/Plots/Uccle_50years/DataModel/' + plname + '.eps')
    plt.show()
    plt.close()
    plt.close()

######################################################################################################################

# part for using extended predictors
pre_name = 'ilt'
plname = 'Trend_' + pre_name
tag = ''

# predictors = pd.read_csv('/home/poyraden/Analysis/MLR_Uccle/Files/Extended_ilt.csv')
predictors=pd.read_csv('/home/poyraden/Analysis/MLR_Uccle/Files/TotalColumnPredictors_ilt.csv')
predictors = predictors.drop(['AO', 'pre_tropop', 'temp_100', 'temp_500', 'EAWR', 'NAO'], axis = 1)

# try new predictors
# predictors= pd.read_csv('/home/poyraden/Analysis/MLR_Uccle/Files/NewPredictors_ilt.csv')

l2 = predictors['Unnamed: 0'].tolist()

print('l2', l2[0:5])

predictors.rename(columns={'Unnamed: 0': 'date'}, inplace=True)
# predictors['date'] = pd.to_datetime(predictors['date'], format='%Y-%m')
predictors.set_index('date', inplace=True)

# For Brewer Mast
predictors = predictors.loc['1971-07-01':'2018-12-01']

# only to plot data/model for monthly means
# predictors = add_seasonal_components(predictors, {'pre_const': 4, 'post_const':4, 'gap_cons':4})

uccle = pd.read_csv('/home/poyraden/Analysis/MLR_Uccle/Files/TotalOzone_monthlymean.csv')

# uccle.rename(columns={'Unnamed: 0':'date'}, inplace=True)
#uccle['date'] =  dates
uccle['dateindex'] = pd.to_datetime(uccle['date'], format='%Y-%m')
print(uccle.dtypes)
# uccle = uccle['dateindex'].resample('6M').mean()

uccle.set_index('date', inplace=True)
print('uccle', len(uccle), list(uccle))

l1 = uccle.index.tolist()

print('l1', l1[0:5])

uccle['monthly_mean'] = uccle['mean'] - uccle['anamoly']
uccle['rel_anamoly'] = (uccle['mean'] - uccle['monthly_mean'])/ uccle['monthly_mean']

print('predictors', len(predictors), list(predictors))

print(uccle.index)

uccle_pre = uccle.loc['1971-07-01':'1996-12-01']
uccle_post = uccle.loc['2000-02-01':'2018-12-01']

mean_pre = np.nanmean(uccle_pre['mean'].values)
mean_post = np.nanmean(uccle_post['mean'].values)

common_dates12 = list(set(l1).intersection(set(l2)))

# only for total column ilt
uccle = uccle[uccle.index.isin(common_dates12)]
predictors = predictors[predictors.index.isin(common_dates12)]

print(len(uccle), len(predictors))
predictors, uccle = pd.DataFrame.align(predictors, uccle, axis=0)

print('last predictor', predictors[0:3])

uYra = uccle['rel_anamoly'].values
uY = uccle['mean'].values
uX = predictors.values
print('uX',uX)
print('uY', uY)
regression_output = mzm_regression(uX, uY)
param_list = dict(zip(list(predictors), regression_output['gls_results'].params))
error_list = dict(zip(list(predictors), regression_output['gls_results'].bse))

plotmlr_perkm(uccle.dateindex,uY,regression_output['fit_values'],'Total O3 Monthly Means','TotalOzone_totalcolumnilt')
# plotmlr_perkm(uccle.dateindex,uYra,regression_output['fit_values'],'Total O3 Monthly Means','TotalOzone_totalcolumnilt')

plotresidual(uccle.dateindex,regression_output['residual'],'residualmean','Residual_TotalOzone_totalcoulumnilt')

# # for relative anamoly
#
# regression_outputra = mzm_regression(uX, uYra)
# param_listra = dict(zip(list(predictors), regression_outputra['gls_results'].params))
# error_listra = dict(zip(list(predictors), regression_outputra['gls_results'].bse))
#
# plotmlr_perkm(uccle.dateindex,uYra,regression_outputra['fit_values'],'Total O3 Relative Monthly Anamolies','Relative_ilt')
# plotresidual(uccle.dateindex,regression_outputra['residual'],'residualmean','Residual_Relative_ilt')

trend_pre=param_list['linear_pre']
trend_pre_err=error_list['linear_pre']
trend_post=param_list['linear_post']
trend_post_err=error_list['linear_post']

# for ra

# trend_prera = param_listra['linear_pre']
# trend_pre_errra = error_listra['linear_pre']
# trend_postra = param_listra['linear_post']
# trend_post_errra = error_listra['linear_post']

# # for % in decade for relative monthly anamoly
# trend_pre=trend_pre * 100
# trend_pre_err=2 * trend_pre_err * 100
# trend_post=trend_post * 100
# trend_post_err=2 * trend_post_err * 100
#
#
# # for % in decade for  montly means
# trend_pre=trend_pre * 100 /mean_pre
# trend_pre_err=2 * trend_pre_err * 100/mean_pre
# trend_post=trend_post * 100/ mean_post
# trend_post_err=2 * trend_post_err * 100/mean_post
# #

# # # for  in decade for  montly means
# trend_pre=trend_pre  /mean_pre
# trend_pre_err=2 * trend_pre_err /mean_pre
# trend_post=trend_post / mean_post
# trend_post_err=2 * trend_post_err/mean_post
# #

print('trend_pre', trend_pre, ' err', trend_pre_err)
print('trend_post', trend_post, ' err', trend_post_err)

print('yearly')
print('pre', trend_pre/10)
print('post', trend_post/10)

print('monthl;y')
print('pre', trend_pre/120)
print('post', trend_post/120)

fig, ax = plt.subplots()

predictors[['linear_pre', 'linear_post']].plot(figsize=(16, 6))

# plt.title('Uccle 1969-2018')
# plt.xlabel('Ozone trend [%/dec]')
# plt.ylabel('Altitude [km]')
# plt.xlim(-10, 10)
# plt.ylim(0,32)
# ax.axvline(x=0, color='grey', linestyle='--')
#
# ax.tick_params(axis='both', which='both', direction='in')
# ax.yaxis.set_ticks_position('both')
# ax.xaxis.set_ticks_position('both')
# # ax.yaxis.set_minor_locator(AutoMinorLocator(5))
# # ax.xaxis.set_minor_locator(AutoMinorLocator(5))
# ax.set_xticks([-10,-5,0,5,10])
#
#
# eb1 = ax.errorbar(trend_pre, uccle.dateindex, xerr=trend_pre_err, label='pre-1997', color='red', linewidth=1)
# eb1[-1][0].set_linestyle('--')
# eb2 = ax.errorbar(trend_post, uccle.dateindex, xerr=trend_post_err, label='post-2000', color='limegreen', linewidth=1)
# eb2[-1][0].set_linestyle('--')
#
# ax.legend(loc='upper right', frameon=True, fontsize='small')
#
# plt.show()

# print('decade rel anamoly')
# print('pre', trend_prera * 100)
# print('post', trend_postra * 100)
#
# print('month rel anamoly')
# print('pre', trend_prera/120)
# print('post', trend_postra/120 )

# didx = pd.DatetimeIndex(start ='1971-07-01', end = '2018-12-01', freq ='MS')
#
# print('year', didx.year)
# relative anamoly
# trend_pre -1.39265258923  err 0.986189844927
# trend_post 1.88317756816  err 1.52182046731

# plt.plot(didx.year,regression_output['fit_values'] )

# plt.plot(uccle.dateindex,regression_output['fit_values'],label='Model',color='black',linewidth=0.75)
# plt.plot(uccle.dateindex, uX * trend_pre,label='Model',color='red',linewidth=0.75)
# plt.plot(didx.year,regression_output['fit_values'] )

# y = 2.7449 + didx.year*trend_pre/120
# y = didx.year*trend_pre/120

# print(y)

# y1 = 331
# t = -0.0411149462777
# post =  0.0324304947709
#
# x = [0] * 570
# xt = [0] * 570
# y = [0] * 570
#
# for i in range(570):
#     xt[i] = i
#     if i < 294:
#         x[i] = i
#         y[i] = y1 + (t*x[i])
#     if ( (i == 294) & (i < 333)):
#         x[i]= i
#         y[i]= y1
#     if i > 333:
#         x[i] = i
#         y[i] = y1 + (post *x[i])
#
#
#
#
#
# #
# #
# #
# fig, ax = plt.subplots()
#
# color = 'tab:red'
# ax.set_xlabel('time (s)')
# ax.set_ylabel('exp', color=color)
# plt.plot(uccle.dateindex,regression_output['fit_values'],label='Model',color='black',linewidth=0.75)
# plt.plot(xt,regression_output['fit_values'],label='Model',color='black',linewidth=0.75)

# ax.tick_params(axis='y', labelcolor=color)
#
# ax2 = ax.twinx()  # instantiate a second axes that shares the same x-axis
#
# color = 'tab:blue'
# # ax2.set_ylabel('sin', color=color)  # we already handled the x-label with ax
# plt.plot(x,y)
# # ax2.tick_params(axis='y', labelcolor=color)
#
# # fig.tight_layout()  # otherwise the right y-label is slightly clipped

# pt = pd.pivot_table(uccle, index=uccle.index.year, columns=uccle.index.month,
#                     aggfunc='sum')
#
# print('pt[0:10', pt[-0:20])
#
# print(pt.index)
# pt.columns = pt.columns.droplevel()
#
# ticklabels = [datetime.date(1971,item,1).strftime('%Y') for item in pt.index]
# print('tick labels', ticklabels)
#
# ax.set_xticks(np.arange(0,47))
# ax.set_xticklabels(ticklabels) #add monthlabels to the xaxis

plt.show()
#
# # plt.plot(uccle.dateindex,regression_output['fit_values'],label='Model',color='black',linewidth=0.75)
# # plt.plot(x,y)
# # plt.plot(uccle.dateindex, y ,label='Model',color='red',linewidth=0.75)
#
#
# # plt.plot(didx.year,y)
#


#!bin/python

import sys, pygame, time

import numpy as np

from sprites import *

from engine import *

from player import *

Engine.collision_pairs.append(["demons","players"])
Engine.collision_pairs.append(["demons","draw"])

@Engine.addGroup
class Demons(pygame.sprite.Group):
    name="demons"

Engine.collision_pairs.append(("demons","planets"))
@Engine.addEntity
class Demon(GravitySprite,AnSprite):
    name="demon"
    groups=["draw","demons"]
    i_w,i_h =  np.array( pygame.image.load("demon_idle.png").get_rect().size )*0.3
    d_w, d_h = np.array(pygame.image.load("demon_death.png").get_rect().size)*0.3
    animations={"idle":Animation(pygame.transform.scale(pygame.image.load("demon_idle.png"),(int(i_w),int(i_h))),8,1,scale=1),
                "death":Animation(pygame.transform.scale(pygame.image.load("demon_death.png"),(int(d_w),int(d_h))),5,1,scale=0.3)}
    def __init__(self,kargs):
        kargs["mass"]=50
        GravitySprite.__init__(self,kargs)
        AnSprite.__init__(self,{"animations":self.animations,"current_animation":"idle"})
        self.speed = numpy.array([1.0, 1.0])
        self.velocity = numpy.array([0.0,0.0])
        self.theta = 0
        self.angVel = 0
        self.alive = True
        self.restitution=0.5

    def update(self,dt,events,collisions):
        HEIGHT = 1
        GRAVITY = 5
        max_speed = 250
        ACCELERATION = 10
        AnSprite.update(self,dt,events,collisions)
        self.calculateGravity()
        #seek player
        mult=0.1
        if(len(Engine.groups["players"].sprites())>0):
            self.acceleration +=(Engine.groups["players"].sprites()[0].pos - self.pos)*mult*self.speed
        dx,dy = self.acceleration
        ang = np.arctan2(dy,dx)
        ang = 360 * (-ang+np.pi/2)/(2*np.pi)
        self.angle = ang - 90
        #max speed
        if(np.linalg.norm(self.velocity) > max_speed):
            self.velocity = self.velocity/np.linalg.norm(self.velocity)*max_speed
        #kill player on contact
        if self in collisions:
            for x in collisions[self]:
                    if isinstance(x, Engine.entities["player"]) and pygame.sprite.collide_mask(x, self):
                        x.alive = False
                    if isinstance(x,Engine.entities["bullet"]) and pygame.sprite.collide_mask(x,self):
                        self.alive = False
                    if isinstance(x,Planet):
                        if isinstance(x,BlackHole):
                            self.alive = False
                        elif((x.pos-self.pos).dot(self.velocity)>0):
                            p = pygame.sprite.collide_mask(self,x)
                            if p:
                                #bounce off
                                d = x.pos - self.pos
                                d /= np.linalg.norm(d)
                                r = self.velocity.dot(d)*(-1-self.restitution)
                                self.velocity +=d*r
        speed= np.linalg.norm( self.velocity )
        if speed> max_speed:
            self.velocity *= max_speed/speed
        self.move(dt)
        if not self.alive:
            #pentagram
            d_w, d_h = np.array(pygame.image.load("demon_death.png").get_rect().size) * 0.1
            death = pygame.transform.scale(pygame.image.load("demon_death.png"), (int(d_w), int(d_h)))
            Engine.new("particle", image = death,xtile = 5,ytile = 1, mass = 10, fps = 10,pos = self.pos)
            self.kill()

@Engine.addEntity
class DemonSpawner(pygame.sprite.Sprite):
    name="spawner"
    groups = ["base"]
    num_demons = 5
    t = 0
    t2 = 0
    def update(self,dt,events,col):
        self.t2 += dt
        if ( self.t2>=5):
            self.t2 = 0
            for i in range(0, 10):
                if (len(Engine.groups["players"].sprites()) > 0):
                    x, y = Engine.groups["players"].sprites()[0].pos
                    r = numpy.random.uniform(1300,3000)
                    theta = numpy.random.uniform(90,200)
                    x += r*numpy.cos(theta)
                    y += r*numpy.sin(theta)
                    Engine.new("demon", pos=[x, y])
        self.t += dt


import seaborn as sns
import numpy as np
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import matplotlib.cbook as cbook
import datetime

sns.set_style("darkgrid")

df = pd.read_csv("ibex_35.csv")

df = df.dropna()

log_returns = np.diff(np.log(df["Close"].values))
date = df["Date"].values[1:]
date = np.array([datetime.datetime.strptime(x , "%Y-%m-%d") for x in date])

years = mdates.YearLocator()
months = mdates.MonthLocator()
years_fmt = mdates.DateFormatter('%Y')

fig, ax = plt.subplots()
ax.plot(date, log_returns , 'k')
ax.set_ylabel("$R_{(t)}$")
ax.xaxis.set_major_locator(years)
ax.xaxis.set_major_formatter(years_fmt)
ax.xaxis.set_minor_locator(months)
ax.set_title("Daily log returns IBEX-35")
datemin = np.datetime64(date[0], 'Y')
datemax = np.datetime64(date[-1], 'Y') + np.timedelta64(1, 'Y')
ax.set_xlim(datemin, datemax)
idx = np.round(np.linspace(0, len(date)-1 , 8)).astype(int)
ax.format_xdata = mdates.DateFormatter('%Y-%m-%d')
ax.format_ydata = lambda x: '$%1.2f' % x
ax.set_xticks(date[idx])
fig.autofmt_xdate()
plt.savefig("ibexlogreturns.pdf")


from django.shortcuts import get_object_or_404, redirect
from django.urls import reverse, reverse_lazy
from django.contrib.auth.mixins import LoginRequiredMixin
from django.views.generic import TemplateView, DetailView
from django.views.generic.edit import CreateView, UpdateView, DeleteView
from django.views.generic import ListView
from django.contrib.auth.decorators import login_required

from .models import Task, TaskExecution, TaskExecutionFile
from .forms import TaskExecutionForm, TaskForm, UpdateTaskExecutionForm, UpdateTaskForm
from django.utils.decorators import method_decorator

@method_decorator(login_required(login_url="/users/login/"), name="dispatch")
class IndexView(TemplateView):
    template_name = "tasks/index.html"

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)

        sort_by = self.request.GET.get(
            "sort_by", "created_at"
        )  # По умолчанию сортировка по дате создания

        if sort_by == "completed":
            context["tasks"] = Task.objects.all().order_by("-completed", "created_at")
        elif sort_by == "created_at":
            context["tasks"] = Task.objects.all().order_by("created_at")
        elif sort_by == "worker":
            context["tasks"] = Task.objects.all().order_by("worker")
        elif sort_by == "created_by":
            context["tasks"] = Task.objects.all().order_by("created_by")
        elif sort_by == "deadline":
            context["tasks"] = Task.objects.all().order_by("deadline")

        return context

# @method_decorator(login_required(login_url='/users/login/'), name='dispatch')
class TaskDetailView(DetailView):
    model = Task
    template_name = "tasks/task_detail.html"

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context["task_execution_form"] = TaskExecutionForm()
        return context

    def post(self, request, *args, **kwargs):
        task = self.get_object()
        task.completed = not task.completed
        task.save()
        return redirect("tasks:task_detail", pk=task.pk)

class CreateTaskView(LoginRequiredMixin, CreateView):
    model = Task
    form_class = TaskForm
    template_name = "tasks/task_create.html"

    def form_valid(self, form):
        form.instance.created_by = self.request.user
        file = self.request.FILES.get("file")  # получаем загруженный файл, если он есть
        if file:
            form.instance.file = file  # присваиваем файлу значение
        return super().form_valid(form)

    def get_success_url(self):
        return reverse("tasks:index")

@method_decorator(login_required(login_url="/users/login/"), name="dispatch")
class UpdateTaskView(UpdateView):
    model = Task
    form_class = UpdateTaskForm
    template_name = "tasks/task_create.html"

    def get_success_url(self):
        return reverse("tasks:index")

@method_decorator(login_required(login_url="/users/login/"), name="dispatch")
class DeleteTaskView(DeleteView):
    model = Task
    template_name = "tasks/delete_task.html"
    success_url = reverse_lazy("tasks:index")

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context["task"] = self.get_object()
        return context

    def get_success_url(self):
        return reverse_lazy("tasks:index")

@method_decorator(login_required(login_url="/users/login/"), name="dispatch")
class FromMeTasks(LoginRequiredMixin, ListView):
    model = Task
    template_name = "tasks/from_me_tasks.html"
    context_object_name = "tasks"

    def get_queryset(self):
        return Task.objects.filter(created_by=self.request.user)

    # Сортировка задач в профиле
    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        sort_by = self.request.GET.get(
            "sort_by", "created_at"
        )  # По умолчанию сортировка по дате создания

        if sort_by == "completed":
            context["tasks"] = context["tasks"].order_by("-completed", "created_at")
        elif sort_by == "created_at":
            context["tasks"] = context["tasks"].order_by("created_at")
        elif sort_by == "worker":
            context["tasks"] = context["tasks"].order_by("worker")
        elif sort_by == "created_by":
            context["tasks"] = context["tasks"].order_by("created_by")
        elif sort_by == "deadline":
            context["tasks"] = context["tasks"].order_by("deadline")

        return context

# @method_decorator(login_required(login_url="/users/login/"), name="dispatch")
class AllTasks(LoginRequiredMixin, ListView):
    model = Task
    template_name = "tasks/all_tasks.html"
    context_object_name = "tasks"

    def get_queryset(self):
        return Task.objects.all()

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        sort_by = self.request.GET.get(
            "sort_by", "created_at"
        )  # По умолчанию сортировка по дате создания

        if sort_by == "completed":
            context["tasks"] = context["tasks"].order_by("-completed", "created_at")
        elif sort_by == "created_at":
            context["tasks"] = context["tasks"].order_by("created_at")
        elif sort_by == "worker":
            context["tasks"] = context["tasks"].order_by("worker")
        elif sort_by == "created_by":
            context["tasks"] = context["tasks"].order_by("created_by")
        elif sort_by == "deadline":
            context["tasks"] = context["tasks"].order_by("deadline")

        return context

# @method_decorator(login_required(login_url="/users/login/"), name="dispatch")
class ForMeTasks(LoginRequiredMixin, ListView):
    model = Task
    template_name = "tasks/for_me_tasks.html"
    context_object_name = "tasks"

    def get_queryset(self):
        return Task.objects.filter(worker=self.request.user)

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        sort_by = self.request.GET.get(
            "sort_by", "created_at"
        )  # По умолчанию сортировка по дате создания

        if sort_by == "completed":
            context["tasks"] = context["tasks"].order_by("-completed", "created_at")
        elif sort_by == "created_at":
            context["tasks"] = context["tasks"].order_by("created_at")
        elif sort_by == "worker":
            context["tasks"] = context["tasks"].order_by("worker")
        elif sort_by == "created_by":
            context["tasks"] = context["tasks"].order_by("created_by")
        elif sort_by == "deadline":
            context["tasks"] = context["tasks"].order_by("deadline")

        return context

# @method_decorator(login_required(login_url='/users/login/'), name='dispatch')
class TaskExecutionCreateView(LoginRequiredMixin, CreateView):
    model = TaskExecution
    form_class = TaskExecutionForm
    template_name = "tasks/task_ex_create.html"

    def get_task(self):
        return get_object_or_404(Task, pk=self.kwargs["task_id"])

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context["task"] = self.get_task()
        context["xfiles"] = context["task"].files.all()
        if context["task"].deadline:
            context["form"].fields["deadline"].initial = context[
                "task"
            ].deadline.strftime("%Y-%m-%dT%H:%M")
        return context

    def form_valid(self, form):
        form.instance.task = self.get_task()
        response = super().form_valid(form)
        task = self.get_task()
        task.deadline = form.instance.deadline
        task.save()

        for each in self.request.FILES.getlist("xfiles"):
            TaskExecutionFile.objects.create(task_execution=self.object, file=each)

        return response

    def get_success_url(self):
        return reverse("tasks:task_detail", kwargs={"pk": self.get_task().pk})

@method_decorator(login_required(login_url="/users/login/"), name="dispatch")
class TaskExecutionDetailView(DetailView):
    model = TaskExecution
    template_name = "tasks/task_execution_detail.html"
    context_object_name = "task_execution"

@method_decorator(login_required(login_url="/users/login/"), name="dispatch")
class UpdateTaskExecution(UpdateView):
    model = TaskExecution
    form_class = UpdateTaskExecutionForm
    template_name = "tasks/task_create.html"

    def form_valid(self, form):
        task_execution = form.instance
        task = task_execution.task

        task.deadline = task_execution.deadline
        task.save()

        return super().form_valid(form)

    def get_success_url(self):
        return reverse("tasks:index")

@method_decorator(login_required(login_url="/users/login/"), name="dispatch")
class DeleteTaskExecutionView(DeleteView):
    model = TaskExecution
    template_name = "tasks/delete_task_execution.html"

    def get_object(self, queryset=None):
        task_execution = super().get_object(queryset)

        all_executions = TaskExecution.objects.filter(task=task_execution.task)
        previous_execution = (
            all_executions.exclude(pk=task_execution.pk).order_by("-created_at").first()
        )

        if previous_execution:
            task_execution.task.deadline = previous_execution.deadline
            task_execution.task.save()

        return task_execution

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context["task_execution"] = self.get_object()
        return context

    def get_success_url(self):
        task_execution = self.get_object()
        return reverse_lazy("tasks:task_detail", kwargs={"pk": task_execution.task.pk})


from mdeditor.fields import MDTextField
from solo.models import SingletonModel

from django.db import models
from django.urls import reverse

from TgAdmin.constants import Const

class Chat(models.Model):
    class Meta:
        verbose_name = 'Название чата'
        verbose_name_plural = 'Названия чатов'

    chat_id = models.CharField(
        max_length=255,
        verbose_name='Id группы',
        unique=True
    )
    name = models.CharField(
        max_length=255,
        verbose_name='Название группы'
    )
    url_link = models.CharField(
        max_length=255,
        verbose_name='Ссылка на тп/магазин',
        choices=Const.LINK,
        default='tp',
        help_text='Выбрать ссылку для перехода пользователя, после нажатия'
                  ' на клавишу "согласен"'
    )
    # здесь language нужен для группового чата
    language = models.CharField(
        max_length=2,
        choices=Const.LANGUAGE,
        default='ru',
        verbose_name='Язык'
    )
    url_chat = models.URLField(
        max_length=255,
        verbose_name='Ссылка на чат',
        blank=True,
        null=True,
        help_text='По умолчанию добавляется ссылка чата. Если измените ссылку-приглашение, то'
                  ' здесь необходимо изменить ссылку'
    )

    def __str__(self):
        return f"chat_id: {self.chat_id}; name: {self.name}"

    def get_absolute_url(self):
        return reverse('chat_edit', args=[str(self.id)])

class Users(models.Model):
    class Meta:
        verbose_name = 'Пользователь телеграма'
        verbose_name_plural = 'Пользователи телеграма'
        ordering = ['-date_joined']
        unique_together = [['iduser', 'chat_id']]

    iduser = models.CharField(
        max_length=255,
        verbose_name='ID пользователя',
        db_index=True
    )
    username = models.CharField(
        max_length=255,
        verbose_name='Username пользователя',
        blank=True,
        null=True,
    )
    fullname = models.CharField(
        max_length=255,
        verbose_name='Полное имя пользователя',
        blank=True,
        null=True
    )
    date_joined = models.DateTimeField(
        auto_now_add=True,
        verbose_name='Дата добавления пользователя'
    )
    state = models.CharField(
        max_length=1,
        choices=Const.STATE,
        default='1',
        db_index=True
    )
    chat_id = models.ForeignKey(
        Chat,
        on_delete=models.SET_NULL,
        null=True,
        related_name='users'
    )
    is_banned = models.BooleanField(default=False)
    # Здесь language нужен для техподдержки
    language = models.CharField(
        max_length=2,
        choices=Const.LANGUAGE_SUPPORT,
        default='no'
    )
    message_id = models.CharField(
        max_length=255,
        verbose_name="Id сообщения при вступлении в группу",
        blank=True,
        null=True
    )

    def __str__(self):
        return f"id-{self.iduser}: username-{self.username}, fullname-{self.fullname}"

def user_directory_path(instance, filename):
    return '{0}'.format(filename)

class Data(SingletonModel):
    class Meta:
        verbose_name = 'Данные для изменения'

    text_before_btn = MDTextField(
        verbose_name='Текст рядом с кнопками (русский)',
        default='Текст для приветствия новых пользователей в чате на русском языке'
    )
    text_before_btn_en = MDTextField(
        verbose_name='Текст рядом с кнопками (english)',
        default='Текст для приветствия новых пользователей в чате на английском языке'
    )
    image = models.ImageField(
        blank=True,
        null=True,
        upload_to=user_directory_path,
        verbose_name='Фотография для текста'
    )
    btn1 = models.CharField(
        max_length=255,
        verbose_name='Кнопка 1 (русский)',
        default='Согласен'
    )
    btn1_en = models.CharField(
        max_length=255,
        verbose_name='Кнопка 1 (english)',
        default='Agree'
    )
    btn2 = models.CharField(
        max_length=255,
        verbose_name='Кнопка 2 (русский)',
        default='Отклонить'
    )
    btn2_en = models.CharField(
        max_length=255,
        verbose_name='Кнопка 2 (english)',
        default='Reject'
    )
    faq = models.TextField(
        verbose_name='FAQ для техподдержки',
        default=Const.TEXT_FOR_FAQ
    )
    faq_en = models.TextField(
        verbose_name='FAQ english для техподдержки',
        default=Const.TEXT_FOR_FAQ_EN
    )
    text_mailing = models.TextField(
        verbose_name='Текс для рассылки в техподдержке',
        default='Текст для рассылки'
    )
    id_admin = models.CharField(
        max_length=100,
        verbose_name='ID пользователя',
        help_text='ID пользователя, кто будет добавлять чаты в БД',
        default='',
    )

    def __str__(self):
        return 'Данные для изменения'

class TempDataSupport(models.Model):
    class Meta:
        verbose_name = 'Временные данные техподдержки'
        verbose_name_plural = 'Временные данные техподдержки'

    id_message = models.CharField(
        max_length=255,
        verbose_name='ID сообщения в техподдержке'
    )
    id_user = models.CharField(
        max_length=255,
        unique=True,
        verbose_name='ID пользователя'
    )
    text = models.TextField(
        verbose_name='Текст сообщения',
        blank=True,
        null=True
    )

    def __str__(self):
        return 'Временные данные техподдержки'

class TempDataBtn(models.Model):
    class Meta:
        verbose_name = 'Временные данные для обработчика'
        verbose_name_plural = 'Временные данные для обработчика'

    id_message = models.CharField(
        max_length=255,
        verbose_name="ID сообщения в техподдержке"
    )
    id_user = models.CharField(
        max_length=255,
        unique=True,
        verbose_name='ID пользователя'
    )

    def __str__(self):
        return 'Временные данные для обработчика'


## Simple Linear Regression

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

## Importing the DataSets

df = pd.read_csv('Salary_Data.csv')
X = df.iloc[:, 0:1].values
y = df.iloc[:, -1:].values

# Training the Data

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.8)

X_train.shape
X_test.shape

'''
# Feature Scaling

from sklearn.preprocessing import StandardScaler
sc_X = StandardScaler()

X_train = sc_X.fit_transform(X_train)
X_test = sc_X.transform(X_test)

sc_y = StandardScaler()
y_train = sc_y.fit_transform(y_train)
'''

##### Using Regression Technique

# Impoting the Regression Techniques
from sklearn.linear_model import LinearRegression
lr = LinearRegression()

# Fitting the model
lr.fit(X_train, y_train)

# predicting the model
y_pred = lr.predict(X_test)

# Accuracy of the model
lr.score(X_test, y_test)

##### Visualization

#Training Visualization
plt.scatter(X_train, y_train, color = 'red')
plt.plot(X_train, lr.predict(X_train), color = 'blue')
plt.title('Salary v/s Expereice')
plt.xlabel('Experience in Years')
plt.ylabel('Salary')
plt.show()

#Testing Visualization
plt.scatter(X_test, y_test, color = 'yellow')
plt.plot(X_test, lr.predict(X_test), color = 'blue')
plt.title('Salary v/s Expereice')
plt.xlabel('Experience in Years')
plt.ylabel('Salary')
plt.show()


import requests
from bs4 import BeautifulSoup
url = 'https://conjugator.reverso.net/conjugation-german-verb-%20machen.html'
resp = requests.get(url)
soup = BeautifulSoup(resp.text, features="lxml", from_encoding='utf-8')

divTag=soup.find_all("div", {"class": "blue-box-wrap"})
#person=divTag.find_all("i", {"class": "graytxt"})

print("conjugations found: " + str(len(divTag)))

for tense in range(len(divTag)):
    print('----------------------------')
    print ("tense: " + str(tense))
    graytxt = divTag[tense].find_all("i", {"class": "graytxt"})
    auxgraytxt = divTag[tense].find_all("i", {"class": "auxgraytxt"})
    particletxt = divTag[tense].find_all("i", {"class": "particletxt"})
    verbtxt = divTag[tense].find_all("i", {"class": "verbtxt"})
    auxgraytxt = divTag[tense].find_all("i", {"class": "auxgraytxt"})
    print(divTag[tense].find("p").text)
    print("graytxt: " + str(len(graytxt)))
    print("auxgraytxt: " + str(len(auxgraytxt)))
    print("particletxt: " + str(len(particletxt)))
    print("verbtxt: " + str(len(verbtxt)))
    for x in range(6):
        conjugation=""
        if len(graytxt):
            conjugation = conjugation + ' ' + graytxt[x].text
        if len(auxgraytxt):
            conjugation  = conjugation  + ' ' + auxgraytxt[x].text
        if len(particletxt):
            conjugation  = conjugation  + ' ' + particletxt[x].text
        if len(verbtxt):
            conjugation  = conjugation   + '' + verbtxt[x].text
        #if len(auxgraytxt):
        #     conjugation  = conjugation   + ' ' + auxgraytxt[x].text
        #print(len(auxgraytxt))
        print(conjugation)


from pandas import DataFrame
from typing import Callable, Union

from factor.base import BaseFactor
from data_manager.TradeHoldDataManager import TradeHoldDataManager

class BaseTradeHoldFactor(BaseFactor):
    """
    成交持仓因子基类
    """
    def __init__(self, **params) -> None:
        """Constructor"""
        super().__init__(**params)

        self.trade_hold_data_manager: TradeHoldDataManager = TradeHoldDataManager()

    def get_trade_hold_data_by_rank(self, rank_by: str = 'volume', group_by_symbol: bool = True) -> DataFrame:
        """
        根据排名标准获取成交持仓数据

        Parameters
        ----------
        rank_by: str, default volume
                排名标准, volume, long, short

        group_by_symbol: bool, default True
                        是否根据品种聚合

        Returns
        -------
        data: 成交持仓数据
        """
        data: DataFrame = self.trade_hold_data_manager.get_trade_hold_data_by_rank(rank_by=rank_by, group_by_symbol=group_by_symbol)
        return data

    def get_groupby_field(self, field: str, func: Union[str, Callable]) -> DataFrame:
        """
        获取根据各合约按日聚合而成以品种为单位的DataFrame

        Parameters
        ----------
        field: str
                字段

        func: Union[str, Callable]
                聚合方式

        Returns
        -------

        """
        daily_data: DataFrame = self.daily_data_manager.daily_data
        if isinstance(func, str):
            result: DataFrame = getattr(daily_data.groupby(['datetime', 'underlying_symbol'])[field], func)()
            result = result.to_frame(field).reset_index()
        elif isinstance(func, Callable):
            result: DataFrame = daily_data.groupby(['datetime', 'underlying_symbol'])[field].apply(func)
            result = result.to_frame(field).reset_index()
        else:
            raise TypeError("func must be a string or a function!")
        return result

    def compute_factor(self) -> DataFrame:
        """计算因子"""
        pass


"""empty message

Revision ID: b5658f866fa4
Revises: a5a5e318846c
Create Date: 2021-06-23 01:56:29.504114

"""
from alembic import op
import sqlalchemy as sa
from forms import genres

# revision identifiers, used by Alembic.
revision = 'b5658f866fa4'
down_revision = 'a5a5e318846c'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('venues_genres')
    op.drop_table('artists_genres')
    op.add_column('artists', sa.Column('genres', sa.ARRAY(sa.String())))
    op.add_column('venues', sa.Column('genres', sa.ARRAY(sa.String())))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('venues', 'genres')
    op.drop_column('artists', 'genres')
    op.create_table(
        'artists_genres',
        sa.Column(
            'artist_id', sa.INTEGER(), autoincrement=False, nullable=False
        ),
        sa.Column(
            'genre_id', sa.INTEGER(), autoincrement=False, nullable=False
        ),
        sa.ForeignKeyConstraint(
            ['artist_id'],
            ['artists.id'],
            name='artists_genres_artist_id_fkey'
        ),
        sa.ForeignKeyConstraint(
            ['genre_id'], ['genres.id'], name='artists_genres_genre_id_fkey'
        ),
        sa.PrimaryKeyConstraint(
            'artist_id', 'genre_id', name='Artist_Genres_pkey'
        )
    )
    op.create_table(
        'venues_genres',
        sa.Column(
            'venue_id', sa.INTEGER(), autoincrement=False, nullable=False
        ),
        sa.Column(
            'genre_id', sa.INTEGER(), autoincrement=False, nullable=False
        ),
        sa.ForeignKeyConstraint(
            ['genre_id'], ['genres.id'], name='venues_genres_genre_id_fkey'
        ),
        sa.ForeignKeyConstraint(
            ['venue_id'], ['venues.id'], name='venues_genres_venue_id_fkey'
        ),
        sa.PrimaryKeyConstraint(
            'venue_id', 'genre_id', name='Venue_Genres_pkey'
        )
    )
    # ### end Alembic commands ###


import math

import torch
import torch.nn as nn
import numpy as np

from utils.tools import get_mask_from_lengths, pad, reparameterize

from .blocks import (
    Flip,
    LinearNorm,
    ConvBlock,
    ConvTransposeBlock,
    NonCausalWaveNet,
)

class VariationalGenerator(nn.Module):
    """ Variational Generator """

    def __init__(self, preprocess_config, model_config):
        super(VariationalGenerator, self).__init__()

        n_mel_channels = preprocess_config["preprocessing"]["mel"]["n_mel_channels"]
        d_model = model_config["transformer"]["encoder_hidden"]
        encoder_layer = model_config["variational_generator"]["encoder_layer"]
        decoder_layer = model_config["variational_generator"]["decoder_layer"]
        conv_kernel_size = model_config["variational_generator"]["conv_kernel_size"]
        conv_stride_size = model_config["variational_generator"]["conv_stride_size"]
        encoder_decoder_hidden = model_config["variational_generator"]["encoder_decoder_hidden"]
        # encoder_decoder_dropout = model_config["variational_generator"]["encoder_decoder_dropout"]
        latent_hidden = model_config["variational_generator"]["latent_hidden"]
        flow_layer = model_config["variational_generator"]["vp_flow_layer"]
        flow_hidden = model_config["variational_generator"]["vp_flow_hidden"]
        flow_kernel = model_config["variational_generator"]["vp_flow_kernel"]

        dilation = 1
        self.padding_size = int(
            dilation * (conv_kernel_size - conv_stride_size) / 2)
        self.latent_hidden = latent_hidden
        self.conv_kernel_size = conv_kernel_size
        self.conv_stride_size = conv_stride_size

        self.n_mel_channels = n_mel_channels

        self.enc_conv = ConvBlock(
            n_mel_channels,
            encoder_decoder_hidden,
            conv_kernel_size,
            stride=conv_stride_size,
            padding=self.padding_size,
            dilation=dilation,
            w_init_gain="linear",
            # dropout=encoder_decoder_dropout,
            activation=None,
            layer_norm=False,
        )
        self.cond_layer_e = torch.nn.utils.weight_norm(
            torch.nn.Conv1d(
                d_model,
                2*encoder_decoder_hidden*encoder_layer,  # d_model
                kernel_size=conv_kernel_size,
                stride=conv_stride_size,
                padding=self.padding_size,
                dilation=dilation), name='weight')
        # self.cond_layer_e_prj = LinearNorm(
        #     d_model, 2*encoder_decoder_hidden*encoder_layer)
        self.enc_wn = NonCausalWaveNet(
            encoder_decoder_hidden,
            conv_kernel_size,
            1,
            encoder_layer,
            d_model,
        )
        self.latent_enc_prj = LinearNorm(
            encoder_decoder_hidden, latent_hidden * 2)
        self.cond_layer_f = nn.Conv1d(
            d_model,
            encoder_decoder_hidden,
            kernel_size=conv_kernel_size,
            stride=conv_stride_size,
            padding=self.padding_size,
            dilation=dilation,
        )
        self.flow = VPFlow(
            latent_hidden,
            flow_hidden,
            flow_kernel,
            1,
            flow_layer,
            gin_channels=d_model
        )
        self.latent_dec_prj = LinearNorm(latent_hidden, encoder_decoder_hidden)
        self.cond_layer_d = torch.nn.utils.weight_norm(
            torch.nn.Conv1d(
                d_model,
                2*encoder_decoder_hidden*decoder_layer,  # d_model
                kernel_size=conv_kernel_size,
                stride=conv_stride_size,
                padding=self.padding_size,
                dilation=dilation), name='weight')
        # self.cond_layer_d_prj = LinearNorm(
        #     d_model, 2*encoder_decoder_hidden*decoder_layer)
        self.dec_wn = NonCausalWaveNet(
            encoder_decoder_hidden,
            conv_kernel_size,
            1,
            decoder_layer,
            d_model,
        )
        self.dec_conv = ConvTransposeBlock(
            encoder_decoder_hidden,
            n_mel_channels,
            conv_kernel_size,
            stride=conv_stride_size,
            padding=self.padding_size,
            output_padding=self.conv_stride_size - 1,  # for preserving sequence length
            dilation=dilation,
            w_init_gain="linear",
            # dropout=encoder_decoder_dropout,
            activation=None,
            layer_norm=False,
        )
        self.residual_layer = LinearNorm(n_mel_channels, d_model)

    def pad_input(self, x):
        """
        x -- [batch_size, max_time, dim]
        """
        if self.conv_stride_size == 1:
            return x
        pad_size = x.shape[1] + 2 * self.padding_size - \
            1 * (self.conv_kernel_size - 1)
        pad_size = pad_size % self.conv_stride_size
        if pad_size > 0:
            pad = torch.zeros(x.shape[0], pad_size, x.shape[2]).to(
                device=x.device, dtype=x.dtype)
            x = torch.cat((x, pad), dim=1)
        return x

    def get_conv_mask(self, mel_len, tgt_max_len, mel_mask):
        """
        mel_len -- [batch_size,]
        mask_conv -- [batch_size, max_time]
        """
        if max(mel_len).item() == tgt_max_len:
            return mel_mask
        mel_len_conv = []
        for ml in mel_len:
            pad_size = ml + 2 * self.padding_size - \
                1 * (self.conv_kernel_size - 1)
            pad_size = pad_size % self.conv_stride_size
            ml_conv = math.floor(((ml + pad_size) + 2 * self.padding_size - 1 * (
                self.conv_kernel_size - 1)) / self.conv_stride_size + 1)
            # no way to remove this?
            mel_len_conv.append(min(ml_conv, tgt_max_len))
        mask_conv = get_mask_from_lengths(torch.tensor(
            mel_len_conv, dtype=mel_len.dtype, device=mel_len.device))
        return mask_conv

    def trim_output(self, x, max_len):
        return x[:, :max_len]

    def forward(self, mel, mel_len, mel_mask, h_text):
        """
        mel -- [batch_size, max_time, n_mels]
        mel_len -- [batch_size,]
        mel_mask -- [batch_size, max_time]
        h_text -- [batch_size, max_time, dim]
        """
        h_text = self.pad_input(h_text)
        mel = self.pad_input(mel)

        # Prepare Conditioner
        # h_text_f = self.cond_layer_f(h_text.transpose(1, 2))  # [B, H, L']
        # h_text_e = self.cond_layer_e_prj(self.cond_layer_e(
        #     h_text.transpose(1, 2)).transpose(1, 2)).transpose(1, 2)  # [B, H, L']
        # h_text_d = self.cond_layer_d_prj(self.cond_layer_d(
        #     h_text.transpose(1, 2)).transpose(1, 2)).transpose(1, 2)  # [B, H, L']
        h_text = h_text.transpose(1, 2)
        h_text_f = self.cond_layer_f(h_text)  # [B, H, L']
        h_text_e = self.cond_layer_e(h_text)  # [B, H, L']
        h_text_d = self.cond_layer_d(h_text)  # [B, H, L']
        mel_mask_conv = self.get_conv_mask(
            mel_len, h_text_f.shape[2], mel_mask).unsqueeze(-1)

        # Encoding
        x = self.enc_conv(mel)
        x = x.contiguous().transpose(1, 2)
        x = self.enc_wn(x, g=h_text_e) * mel_mask_conv.transpose(1, 2)
        x = x.contiguous().transpose(1, 2)
        x = self.latent_enc_prj(x)

        # # Reparameterization
        m_q, logs_q = torch.split(x, self.latent_hidden, dim=-1)
        m_q, logs_q = m_q * mel_mask_conv, logs_q * mel_mask_conv
        z_q = reparameterize(m_q, logs_q) * mel_mask_conv

        # Prior VP FLow
        z_p = self.flow(z_q.transpose(1, 2), x_mask=mel_mask_conv.transpose(
            1, 2), g=h_text_f, reverse=False)

        # Decoding
        x = self.latent_dec_prj(z_q)
        x = x.contiguous().transpose(1, 2)
        x = self.dec_wn(x, g=h_text_d) * mel_mask_conv.transpose(1, 2)
        x = x.contiguous().transpose(1, 2)
        mel_res = self.dec_conv(x)
        mel_res = self.trim_output(
            mel_res, mel_mask.shape[1]) * mel_mask.unsqueeze(-1)
        residual = self.residual_layer(mel_res) * mel_mask.unsqueeze(-1)

        return mel_res, residual, (z_p, logs_q.transpose(1, 2), mel_mask_conv.transpose(1, 2))

    def inference(self, mel_len, mel_mask, h_text):
        """
        mel_len -- [batch_size,]
        mel_mask -- [batch_size, max_time]
        h_text -- [batch_size, max_time, dim]
        """
        batch_size, device, dtype = h_text.shape[0], h_text.device, h_text.dtype
        h_text = self.pad_input(h_text)

        # Prepare Conditioner
        # h_text_f = self.cond_layer_f(h_text.transpose(1, 2))  # [B, H, L']
        # h_text_d = self.cond_layer_d_prj(self.cond_layer_d(
        #     h_text.transpose(1, 2)).transpose(1, 2)).transpose(1, 2)  # [B, H, L']
        h_text = h_text.transpose(1, 2)
        h_text_f = self.cond_layer_f(h_text)  # [B, H, L']
        h_text_d = self.cond_layer_d(h_text)  # [B, H, L']
        mel_mask_conv = self.get_conv_mask(
            mel_len, h_text_f.shape[2], mel_mask).unsqueeze(-1)

        # Sample from Prior
        z_n = torch.randn(h_text_f.shape[0], self.latent_hidden,
                          h_text_f.shape[2]).to(device=h_text_f.device, dtype=h_text.dtype)
        z_q = self.flow(z_n, x_mask=mel_mask_conv.transpose(
            1, 2), g=h_text_f, reverse=True)

        # Decoding
        x = self.latent_dec_prj(z_q.transpose(1, 2))
        x = x.contiguous().transpose(1, 2)
        x = self.dec_wn(x, g=h_text_d) * mel_mask_conv.transpose(1, 2)
        x = x.contiguous().transpose(1, 2)
        mel_res = self.dec_conv(x)
        mel_res = self.trim_output(
            mel_res, mel_mask.shape[1]) * mel_mask.unsqueeze(-1)
        residual = self.residual_layer(mel_res) * mel_mask.unsqueeze(-1)

        return mel_res, residual, None

class VPFlow(nn.Module):
    """ Volume-Preserving Normalizing Flow """

    def __init__(self,
                 channels,
                 hidden_channels,
                 kernel_size,
                 dilation_rate,
                 n_layers,
                 n_flows=4,
                 gin_channels=0):
        super().__init__()
        self.channels = channels
        self.hidden_channels = hidden_channels
        self.kernel_size = kernel_size
        self.dilation_rate = dilation_rate
        self.n_layers = n_layers
        self.n_flows = n_flows
        self.gin_channels = gin_channels

        self.flows = nn.ModuleList()
        for i in range(n_flows):
            self.flows.append(ResidualCouplingLayer(channels, hidden_channels, kernel_size,
                              dilation_rate, n_layers, gin_channels=gin_channels, mean_only=True))
            self.flows.append(Flip())

    def forward(self, x, x_mask=None, g=None, reverse=False):
        if not reverse:
            for flow in self.flows:
                x, _ = flow(x, x_mask, g=g, reverse=reverse)
        else:
            for flow in reversed(self.flows):
                x = flow(x, x_mask, g=g, reverse=reverse)
        return x

class ResidualCouplingLayer(nn.Module):
    def __init__(self,
                 channels,
                 hidden_channels,
                 kernel_size,
                 dilation_rate,
                 n_layers,
                 p_dropout=0,
                 gin_channels=0,
                 mean_only=False):
        assert channels % 2 == 0, "channels should be divisible by 2"
        super().__init__()
        self.channels = channels
        self.hidden_channels = hidden_channels
        self.kernel_size = kernel_size
        self.dilation_rate = dilation_rate
        self.n_layers = n_layers
        self.half_channels = channels // 2
        self.mean_only = mean_only

        self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1)
        self.cond_layer = torch.nn.utils.weight_norm(
            torch.nn.Conv1d(
                gin_channels, 2*hidden_channels*n_layers, 1), name='weight')
        self.enc = NonCausalWaveNet(hidden_channels, kernel_size, dilation_rate,
                                    n_layers, p_dropout=p_dropout, gin_channels=gin_channels)
        self.post = nn.Conv1d(
            hidden_channels, self.half_channels * (2 - mean_only), 1)
        self.post.weight.data.zero_()
        self.post.bias.data.zero_()

    def forward(self, x, x_mask=None, g=None, reverse=False):
        x0, x1 = torch.split(x, [self.half_channels]*2, 1)
        if x_mask is None:
            x_mask = 1
        h = self.pre(x0) * x_mask
        if g is not None:
            g = self.cond_layer(g)
        h = self.enc(h, x_mask=x_mask, g=g)
        stats = self.post(h) * x_mask
        if not self.mean_only:
            m, logs = torch.split(stats, [self.half_channels]*2, 1)
        else:
            m = stats
            logs = torch.zeros_like(m)

        if not reverse:
            x1 = m + x1 * torch.exp(logs) * x_mask
            x = torch.cat([x0, x1], 1)
            logdet = torch.sum(logs, [1, 2])
            return x, logdet
        else:
            x1 = (x1 - m) * torch.exp(-logs) * x_mask
            x = torch.cat([x0, x1], 1)
            return x


#!/usr/bin/python3
# -*- coding: utf-8 -*-

import sys, os
from PyQt5.QtWidgets import QWidget, QDesktopWidget, QApplication, QMainWindow, QFrame, QTableView, QVBoxLayout, QAbstractItemView, QFileDialog, QAction, QMessageBox
from PyQt5 import QtGui, QtCore
import csv
import shutil
import re
import unicodedata
import pprint
import codecs

class MyTableView(QTableView):

    item_dropped = QtCore.pyqtSignal(str, int)

    def __init__(self, parent=None):
        super(MyTableView, self).__init__(parent)

    def dragMoveEvent(self, event):
        index = self.indexAt(event.pos())
        self.selectRow(index.row())
        event.accept()

    def dropEvent(self, event):

        if event.mimeData().hasUrls:
                event.setDropAction(QtCore.Qt.CopyAction)
                event.accept()
                links = []
                for url in event.mimeData().urls():

                    links.append(str(url.toLocalFile()))

                    index = self.indexAt(event.pos())
                    row = index.row()
                    #if not index.isValid():
                        #row = index.row()
                    self.item_dropped.emit(links[0], row)

class MyModel(QtGui.QStandardItemModel):

    def supportedDragActions(self):
        return QtCore.Qt.LinkAction | QtCore.Qt.CopyAction

    def canDropMimeData(self, *args):
        return True

class Example(QMainWindow):

    def __init__(self):
        super().__init__()

        self.root_dir = None
        self.ext = '.pdf'

        #self.resize(300, 400)
        #self.center()
        self.setGeometry(5, 50, 500, 400)

        menubar = self.menuBar()
        menubar.setNativeMenuBar(False)
        file_menu = menubar.addMenu('&File')

        open_action = QAction(QtGui.QIcon(self.resource_path('open.png')), '&Open', self)
        open_action.setShortcut('Ctrl+O')
        open_action.setStatusTip('Open File')
        open_action.triggered.connect(self.open_csv_slot)
        file_menu.addAction(open_action)

        delete_action = QAction('Delete', self)
        delete_action.triggered.connect(self.delete_selected_rows)
        file_menu.addAction(delete_action)

        set_root_dir_action = QAction('Set Root Directory', self)
        set_root_dir_action.triggered.connect(self.set_root_dir)
        file_menu.addAction(set_root_dir_action)

        match_action = QAction('Match', self)
        match_action.triggered.connect(self.match_files)
        file_menu.addAction(match_action)

        main_frame = QFrame()
        self.setCentralWidget(main_frame)
        vbox = QVBoxLayout(main_frame)

        self.tableview = MyTableView()

        self.tableview.setDropIndicatorShown(True)
        self.tableview.setDragEnabled(True)
        self.tableview.setAcceptDrops(True)
        self.tableview.setDragDropMode(QTableView.DragDrop)
        self.tableview.setDefaultDropAction(QtCore.Qt.LinkAction)
        self.tableview.setDropIndicatorShown(True)
        self.tableview.item_dropped.connect(self.item_dropped_slot)
        vbox.addWidget(self.tableview)

        self.model = MyModel()
        self.tableview.setModel(self.model)

        self.setWindowTitle('Center')

        #self.root_dir = ''
        #self.read_csv('')
        #self.match_files()
        self.show()

    def item_dropped_slot(self, path, row):
        #print(path, row)

        if os.path.isdir(path):
            print('dropped folder')
            self.root_dir = path
            return

        ext = os.path.splitext(path)[-1].lower()

        if ext == '.csv':
            self.read_csv(path)

        elif ext == '.pdf':

            self.tableview.clearSelection()

            if row >= 0:

                date_item = self.model.item(row, 0)
                desc_item = self.model.item(row, 1)

                if self.root_dir is None:
                    QMessageBox.information(self, 'Message', "No valid root directory is set")
                    print('No valid root directory is set')
                    return

                if not os.path.isdir(self.root_dir):
                    QMessageBox.information(self, 'Message', "No valid root directory is set")
                    print('No valid root directory is set')
                    return

                date_item.setIcon(QtGui.QIcon(self.resource_path('checkmark.png')))

                for col in range(self.model.columnCount()):

                    self.model.item(row, col).setBackground(QtGui.QBrush(QtGui.QColor(128, 218, 112)))

                new_name = r'{} {}'.format(date_item.text(), desc_item.text().replace("/", ' '))
                #print(new_name.replace("/", '_'))
                self.copy_to_folder(r'{}'.format(path),r'{}.pdf'.format(new_name))

    def match_files(self):

        if self.root_dir is None:
            QMessageBox.information(self, 'Message', "No valid root directory is set")
            print('No valid root directory is set')
            return

        if not os.path.isdir(self.root_dir):
            QMessageBox.information(self, 'Message', "No valid root directory is set")
            print('No valid root directory is set')
            return

        file_list = [p for p in os.listdir(self.root_dir) if os.path.isfile(os.path.join(self.root_dir, p)) and os.path.splitext(p)[1] == self.ext]

        num_rows = self.model.rowCount()

        match_list = []

        for file in file_list:

            for row in range(num_rows):
                date = self.model.index(row, 0).data()
                desc = unicodedata.normalize('NFC', self.model.index(row, 1).data())

                name = r'{} {}'.format(date, desc)

                file_name = unicodedata.normalize('NFC', file)

                if file_name.startswith(name):
                    match_list.append(file_name)
                #if re.search(name, file_name):
                    #match_list.append(file_name)

                    for col in range(self.model.columnCount()):
                        self.model.item(row, col).setBackground(QtGui.QBrush(QtGui.QColor(128, 218, 112)))

    def set_root_dir(self):

        sel_dir = QFileDialog.getExistingDirectory()
        print(sel_dir)
        if sel_dir:
            self.root_dir = sel_dir

    def delete_selected_rows(self):

        reply = QMessageBox.question(self, 'Message', "Delete selected rows?", QMessageBox.Yes | QMessageBox.No, QMessageBox.No)
        if reply == QMessageBox.No:
            return

        selected_indexes =  self.tableview.selectionModel().selectedRows()

        sel_rows = []
        for index in selected_indexes:
            sel_rows.append(index.row())

        sel_rows.sort()
        num_rows = len(sel_rows)

        for i in range(num_rows):
            self.model.removeRow(sel_rows[num_rows-i-1])

    def populate_model(self, header_labels, data_list):

        self.model.clear()
        # set the first 3 headers, by slicing the off
        self.model.setHorizontalHeaderLabels(header_labels[:3])

        for row, data in enumerate(data_list):

            date_item = QtGui.QStandardItem(data[0])
            date_item.setFlags(QtCore.Qt.ItemIsEnabled | QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsDragEnabled | QtCore.Qt.ItemIsDropEnabled)
            self.model.setItem(row,0,date_item)

            desc = '{}'.format(unicodedata.normalize('NFC', data[1]))
            desc_item = QtGui.QStandardItem(desc)
            desc_item.setFlags(QtCore.Qt.ItemIsEnabled | QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsDragEnabled | QtCore.Qt.ItemIsDropEnabled)
            self.model.setItem(row,1,desc_item)

            amount_item = QtGui.QStandardItem(data[2])
            amount_item.setFlags(QtCore.Qt.ItemIsEnabled | QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsDragEnabled | QtCore.Qt.ItemIsDropEnabled)
            self.model.setItem(row,2,amount_item)

        self.tableview.horizontalHeader().setStretchLastSection(True)
        self.tableview.resizeColumnsToContents()

    def resource_path(self, relative_path):
        """ Get absolute path to resource, works for dev and for PyInstaller """
        try:
            # PyInstaller creates a temp folder and stores path in _MEIPASS
            base_path = sys._MEIPASS
        except Exception:
            base_path = os.path.abspath(".")

        return os.path.join(base_path, relative_path)

    def center(self):

        qr = self.frameGeometry()
        cp = QDesktopWidget().availableGeometry().center()
        qr.moveCenter(cp)
        self.move(qr.topLeft())

    def copy_to_folder(self, source_file, new_name):

        if self.root_dir is None:
            QMessageBox.information(self, 'Message', "No valid root directory is set")
            print('No valid root directory is set')
            return

        if not os.path.isdir(self.root_dir):
            QMessageBox.information(self, 'Message', "No valid root directory is set")
            print('No valid root directory is set')
            return

        dest_file = os.path.join(self.root_dir, new_name)

        if os.path.isfile(dest_file):
            reply = QMessageBox.question(self, 'Message', "File exists, overwrite?", QMessageBox.Yes | QMessageBox.No, QMessageBox.No)
            if reply == QMessageBox.No:
                return

        shutil.copy(source_file, dest_file)

    def open_csv_slot(self, *args):
        file_name, selected_filter = QFileDialog.getOpenFileName(None, 'Load csv file', None, filter='CSV (*.csv)')
        if file_name:
            self.read_csv(file_name)

    def read_csv(self, file_name):

        if not os.path.isfile(file_name):
            print('Not a vlid file path')
            return

        if file_name:

            data_list = []
            header_list = None

            with open(file_name, encoding='utf-8') as csvfile:
                dialect = csv.Sniffer().sniff(csvfile.read(1024))
                csvfile.seek(0)
                reader = csv.reader(csvfile, dialect)

                for row_index, row in enumerate(reader):

                    if row_index == 0:
                        header_list = row
                    else:
                        data_list.append(row)

            self.populate_model(header_list, data_list)

if __name__ == '__main__':

    app = QApplication(sys.argv)
    ex = Example()
    sys.exit(app.exec_())


import redis
import time
import datetime
import numpy as np
from bokeh.plotting import figure, output_file, show, HBox, VBox
from bokeh.document import Document

""" 1. Create a redis database object - redis_object
    2. Pull 60 most recent temp values from redis db redis_object.zrange()
    3. Convert the redis data strings into integers with convert_tempstring_to_integers
    6. Pull out the temp data by tank number with  get_temps_by_tank()
    7. Assign the x an y axis values to bokeh lists
    8. Create  bokeh plots and index.html file then copy to /var/www for nginx access
"""

def convert_date_to_epochscore(date_string):

    """conver_date_to_epochscore takes a human readable data and
     converts it to seconds since epoch.  We use that second since
     epoch as a score in the redis database"""

    date_pattern = '%Y/%m/%d %H:%M:%S'
    time_stamp = time.strptime(date_string, date_pattern)
    epoch_seconds = float(time.mktime(time_stamp))
   # epoch_seconds = float(time.mktime(time.strptime(data_string, date_pattern)))
    return epoch_seconds

def convert_epoch_to_date(epoch_int):

    """convert_epoch_to_date takes an seconds to epoch integer and converts
    it to a human readable format"""

    human_readable_date = float(
        datetime.datetime.fromtimestamp(epoch_int).strftime(
        '%Y/%m/%d %H:%M:%S'))
    return human_readable_date

def convert_tempstring_to_floats(temp_string):

    """Clean up the data string from redis and convert the list elements
    to floats"""

    temp_list = temp_string.split(", ")

    first_element = temp_list[0].replace("[", "")
    temp_list[0] = first_element

    last_element = temp_list[-1].replace("]", "")
    temp_list[-1] = last_element

    temp_floats = [float(temp) for temp in temp_list]
    return temp_floats

def convert_redis_data_to_floats(range_data2):

    """convert_redis_data_to_integers using temp_floats to convert the
    entire data from redis into row and reconized by timestamp"""

    raw_temp_list = []

    for data in range_data2:
        temp_floats = convert_tempstring_to_floats(data)
        raw_temp_list.append(temp_floats)
    return raw_temp_list

def get_temps_by_tank(tank_temp_list, num_vals=9):

    """get_temps_by_tank takes a list of redis database range temperatures and
    pulls out column data."""

    # now initialize the return variable
    temps_by_tank = []

    for index in range(num_vals):  #8 temps and 1 epoch second values = 9 range items
        temp_temps = [tmp[index] for tmp in tank_temp_list]  #pull column data
        temps_by_tank.append(temp_temps)
    return temps_by_tank

if __name__ == '__main__':

    # create redis connection
    redis_object = redis.StrictRedis(host='localhost', port=6379, db=0)
    #  query the redis database with a range of dates
    most_recent_data = redis_object.zrange('Temps', -40, -1)
    # convert each row from redis call into floats
    tank_temps = convert_redis_data_to_floats(most_recent_data)
    # pull out the tank temps and group into y axis lists
    temps_by_tank = get_temps_by_tank(tank_temps)
    # pull out the last elment from the temps by tank which is epoch score
    x_axis_time_stamps = temps_by_tank[-1]
    # adjust the timestamps to central time -18000 seconds or 5 hours
    x_axis_time_stamps = [epoch - 18000 for epoch in x_axis_time_stamps]
    #create the x axis for bokeh
    x_axis_time_stamps = [datetime.datetime.fromtimestamp(ts) for ts in x_axis_time_stamps]
    #create date time object to a bokeh time line
    x_axis_time_stamps = [np.datetime64(dt).astype(long)/1000 for dt in x_axis_time_stamps]

    # assign each tank its list of temperatures
    y0_temps = temps_by_tank[0]
    y1_temps = temps_by_tank[1]
    y2_temps = temps_by_tank[2]
    y3_temps = temps_by_tank[3]
    y4_temps = temps_by_tank[4]
    y5_temps = temps_by_tank[5]
    y6_temps = temps_by_tank[6]
    y7_temps = temps_by_tank[7]
    y8_temps = temps_by_tank[8]

    #create the bokeh document to populate with plots and index.html file
    Document = Document()
    output_file("/var/www/index.html")
    # create some attributes to save typing when formating plots
    attrib_dict = dict(width=300, height=300, x_axis_type="datetime")
    line_dict = dict(size=12, alpha=0.5)

    # create the figures that hold each tanks plot
    plot0 = figure(title="Tank 1", **attrib_dict)
    plot1 = figure(title="Tank 2", **attrib_dict)
    plot2 = figure(title="Tank 3", **attrib_dict)
    plot3 = figure(title="Tank 4", **attrib_dict)
    plot4 = figure(title="Tank 5", **attrib_dict)
    plot5 = figure(title="Tank 6", **attrib_dict)
    plot6 = figure(title="Tank 7", **attrib_dict)
    plot7 = figure(title="Tank 8", **attrib_dict)

    # create the lines that go on the figure with attributes
    plot0.line(x_axis_time_stamps, y0_temps, color="black", **line_dict)
    plot1.line(x_axis_time_stamps, y1_temps, color="red", **line_dict)
    plot2.line(x_axis_time_stamps, y2_temps, color="green", **line_dict)
    plot3.line(x_axis_time_stamps, y3_temps, color="blue", **line_dict)
    plot4.line(x_axis_time_stamps, y4_temps, color="violet", **line_dict)
    plot5.line(x_axis_time_stamps, y5_temps, color="blue", **line_dict)
    plot6.line(x_axis_time_stamps, y6_temps, color="red", **line_dict)
    plot7.line(x_axis_time_stamps, y7_temps, color="purple", **line_dict)

    # tell bokeh to create the plots and display them
    show(VBox(HBox(plot0,plot1,plot2,plot3), HBox(plot4,plot5,plot6,plot7)))


import logging

base_config = {
    'trainer': 'multitask',
    'lr': 5e-5, # only for bert for now
    'weight_decay': 0.01,
    'warmup_steps': 100, # number of steps to warm up for learning rate scheduler
    'collapse_domains': True, # whether to load all datasets together or individually
    'epochs': 3,
    'batch_size': 16,
    'valid_freq': 10,
    'save_freq': 100,
    'unfreeze_layers': (10, 11), # layers of bert to unfreeze
    'clip_grad_norm': 1,
    'validation_size': 0, # percentage of train data used for validation data (over split, not over domain)
    'random_state': 42, # random state for reproducibility, currently only used for splitting train and val
    'device': 'cuda',
    'data_dir': 'data/mtl-dataset/',
    'transformer_name': 'bert-base-uncased',
    'domains': ['apparel', 'baby', 'books', 'camera_photo', 'electronics',
                'health_personal_care', 'imdb', 'kitchen_housewares', 'magazines',
                'music', 'software', 'sports_outdoors', 'toys_games', 'video', 'MR', 'dvd'],
    'train_domains':  ['apparel', 'baby', 'books', 'camera_photo', 'electronics',
                'health_personal_care', 'kitchen_housewares', 'magazines',
                'music', 'software', 'toys_games', 'video', 'MR', 'imdb'],
    'val_domains': ['sports_outdoors'],
    'test_domains': ['dvd'],
    'log_level': logging.INFO,
    'log_freq': 100
}

def get_config(base_config, new_config):
    config = dict(base_config)
    for key, value in new_config.items():
        config[key] = value
    return config


#! usr/bin/env python

import wx

class windowClass(wx.Frame):

	def __init__(self, *args, **kwargs):
		super(windowClass, self).__init__(*args, **kwargs)

		self.basicGUI()

	def basicGUI(self):

		panel = wx.Panel(self) # Create text box
		menuBar = wx.MenuBar() # Create menu bar
		fileButton = wx.Menu() # creates single menu - e.g., File button
		editButton = wx.Menu()
		exitItem = fileButton.Append(wx.ID_EXIT, 'Exit', 'Status: Exiting window...')
		menuBar.Append(fileButton, 'File') # Append buttons to menu
		menuBar.Append(editButton, 'Edit')
		wx.TextCtrl(panel, pos = (10,10), size=(500,500), style = wx.TE_MULTILINE) # create text box
		self.SetMenuBar(menuBar)
		self.Bind(wx.EVT_MENU, self.Quit, exitItem)
		self.Maximize()
		self.Show()

	def Quit(self, event):
		self.Close()

def main():
	app = wx.App()
	windowClass(None, title = 'Mein Window')
	app.MainLoop()

if __name__ == '__main__':
	main()


import os

from setuptools import setup, find_packages

import nomos

with open(os.path.join(os.path.dirname(__file__), 'README.rst')) as f:
    readme = f.read()

setup(name='nomos',
      version=nomos.__version__,
      description='nomos',
      long_description=readme,
      author="Thomas Huang",
      url='https://github.com/whiteclover/nomos',
      author_email='lyanghwy@gmail.com',
      packages=find_packages(exclude=["t"]),
      install_requires=['setuptools', 'requests'],
      keywords=["Http web service unit test tool"],
      license="Apache License",
      platforms='any',
      classifiers=(
          "Development Status :: 3 - Alpha",
          "Natural Language :: English",
          "Programming Language :: Python",
          "Programming Language :: Python :: 2.7",
          "Operating System :: OS Independent",
      )
      )