import random
import factory
from faker import Faker
from django.core.files.base import ContentFile
from django.contrib.auth import models as auth_models
from django.utils.timezone import utc
from ninetofiver import models

fake = Faker()

class UserFactory(factory.DjangoModelFactory):
    class Meta:
        model = auth_models.User

    first_name = factory.LazyFunction(fake.first_name)
    last_name = factory.LazyFunction(fake.last_name)
    username = factory.Sequence(lambda n: 'UserName%d' % n)
    email = factory.LazyFunction(fake.name)
    is_staff = False
    is_superuser = False

class AdminFactory(UserFactory):
    is_staff = True
    is_superuser = True

class GroupFactory(factory.DjangoModelFactory):
    class Meta:
        model = auth_models.Group

    name = factory.Sequence(lambda n: 'Group%d' % n)

class CompanyFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.Company

    vat_identification_number = factory.LazyFunction(
        lambda: '%s%s' % (fake.language_code(), fake.md5()[:10])
    )
    name = factory.Sequence(lambda n: 'CompanyName%d' % n)
    address = factory.LazyFunction(fake.address)
    country = factory.LazyFunction(fake.country_code)
    internal = factory.LazyFunction(fake.boolean)

class InternalCompanyFactory(CompanyFactory):
    internal = True

class EmploymentContractTypeFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.EmploymentContractType

    name = factory.Sequence(lambda n: 'EmploymentContractType%d' % n)

class EmploymentContractFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.EmploymentContract

    started_at = factory.LazyFunction(lambda: fake.date_time_this_decade(before_now=True))
    ended_at = factory.LazyFunction(lambda: fake.date_time_this_decade(before_now=False, after_now=True))

class WorkScheduleFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.WorkSchedule

    name = factory.Sequence(lambda n: 'WorkSchedule%d' % n)
    monday = factory.LazyFunction(lambda: random.randint(0, 10))
    tuesday = factory.LazyFunction(lambda: random.randint(0, 10))
    wednesday = factory.LazyFunction(lambda: random.randint(0, 10))
    thursday = factory.LazyFunction(lambda: random.randint(0, 10))
    friday = factory.LazyFunction(lambda: random.randint(0, 10))
    saturday = factory.LazyFunction(lambda: random.randint(0, 10))
    sunday = factory.LazyFunction(lambda: random.randint(0, 10))

class UserInfoFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.UserInfo

    birth_date = factory.LazyFunction(lambda: fake.date_time_this_decade(before_now=True).date())
    gender = factory.LazyFunction(lambda: fake.simple_profile(sex=None)['sex'])
    country = factory.LazyFunction(fake.country_code)

class UserRelativeFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.UserRelative

    name = factory.LazyFunction(fake.name)
    birth_date = factory.LazyFunction(lambda: fake.date_time_this_decade(before_now=True).date())
    gender = factory.LazyFunction(lambda: fake.simple_profile(sex=None)['sex'])
    relation = factory.LazyFunction(fake.word)

class AttachmentFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.Attachment

    name = factory.LazyFunction(fake.word)
    description = factory.LazyFunction(lambda: fake.text(max_nb_chars=200))
    file = factory.LazyFunction(lambda: ContentFile(fake.text(max_nb_chars=200)))

class HolidayFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.Holiday

    name = factory.LazyFunction(fake.word)
    date = factory.LazyFunction(lambda: fake.date_time_this_decade(before_now=True))
    country = factory.LazyFunction(fake.country_code)

class LeaveTypeFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.LeaveType

    name = factory.Sequence(lambda n: 'LeaveType%d' % n)
    description = factory.LazyFunction(lambda: fake.text(max_nb_chars=200))

class LocationFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.Location

    name = factory.Sequence(lambda n: 'Location%d' % n)

class LeaveFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.Leave

    description = factory.LazyFunction(lambda: fake.text(max_nb_chars=200))

class LeaveDateFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.LeaveDate

    starts_at = factory.LazyFunction(lambda: fake.date_time_between(start_date='-1h', end_date='now', tzinfo=utc))
    ends_at = factory.LazyFunction(lambda: fake.date_time_between(start_date='now', end_date='+1h', tzinfo=utc))

class PerformanceTypeFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.PerformanceType

    name = factory.Sequence(lambda n: 'PerformanceType%d' % n)
    description = factory.LazyFunction(lambda: fake.text(max_nb_chars=200))
    multiplier = factory.LazyFunction(lambda: random.randint(0, 3))

class ContractGroupFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.ContractGroup

    name = factory.Sequence(lambda n: 'ContractGroup%d' % n)

class ContractFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.Contract

    name = factory.LazyFunction(fake.word)
    description = factory.LazyFunction(lambda: fake.text(max_nb_chars=200))
    active = factory.LazyFunction(fake.boolean)
    starts_at = factory.LazyFunction(lambda: fake.date_time_this_decade(before_now=True))
    ends_at = factory.LazyFunction(lambda: fake.date_time_this_decade(before_now=False, after_now=True))
    company = factory.SubFactory(InternalCompanyFactory)
    customer = factory.SubFactory(CompanyFactory)

class ProjectContractFactory(ContractFactory):
    class Meta:
        model = models.ProjectContract

    fixed_fee = factory.LazyFunction(lambda: random.randint(0, 9999))

class ConsultancyContractFactory(ContractFactory):
    class Meta:
        model = models.ConsultancyContract

    duration = factory.LazyFunction(lambda: random.randint(0, 9999))
    day_rate = factory.LazyFunction(lambda: random.randint(0, 9999))

class SupportContractFactory(ContractFactory):
    class Meta:
        model = models.SupportContract

    day_rate = factory.LazyFunction(lambda: random.randint(0, 9999))

class ContractRoleFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.ContractRole

    name = factory.Sequence(lambda n: 'ContractRole%d%s' % (n, fake.text(max_nb_chars=200)))
    description = factory.LazyFunction(lambda: fake.text(max_nb_chars=200))

class ContractUserFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.ContractUser

    contract_role = factory.SubFactory(ContractRoleFactory)

class ContractUserGroupFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.ContractUserGroup

class TimesheetFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.Timesheet

    month = factory.LazyFunction(lambda: random.randint(1, 12))
    year = factory.LazyFunction(lambda: random.randint(2000, 3000))

class OpenTimesheetFactory(TimesheetFactory):
    status = models.STATUS_ACTIVE

class WhereaboutFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.Whereabout

    starts_at = factory.LazyFunction(lambda: fake.date_time_between(start_date='-1h', end_date='now', tzinfo=utc))
    ends_at = factory.LazyFunction(lambda: fake.date_time_between(start_date='now', end_date='+1h', tzinfo=utc))
    location = factory.SubFactory(LocationFactory)
    timesheet = factory.SubFactory(OpenTimesheetFactory)

class PerformanceFactory(factory.DjangoModelFactory):
    class Meta:
        model = models.Performance

    date = factory.LazyFunction(lambda: fake.date_time_this_decade().date())
    redmine_id = factory.LazyFunction(lambda: random.randint(0, 3000))

class ActivityPerformanceFactory(PerformanceFactory):
    class Meta:
        model = models.ActivityPerformance

    description = factory.LazyFunction(lambda: fake.text(max_nb_chars=200))
    duration = factory.LazyFunction(lambda: random.randint(1, 24))
    contract_role = ContractRoleFactory

class StandbyPerformanceFactory(PerformanceFactory):
    class Meta:
        model = models.StandbyPerformance


from classes.repository.conectable_mixin import ConnectableMixin
from classes.entity.domain import Domain
from datetime import datetime

class DomainRepository(ConnectableMixin):

    def __init__(self):
        super().__init__()

    def save(self, domain:Domain) -> Domain:
        SQL = """
        INSERT INTO domains (description, deleted_at) VALUES (?, ?)
        """
        domain.id = self.execute_last_row_id(SQL, (domain.description,None))
        if domain.id:
            return domain
        else:
            return None

    def update(self, domain:Domain) -> Domain:
        SQL = """
        UPDATE
            domains
        SET
            description = ?
        WHERE
            id = ? AND deleted_at is null
        """
        affectedRows = self.execute(SQL, (domain.description, domain.id))
        if affectedRows > 0:
            return domain
        else:
            return None

    def delete(self, domain:Domain) -> bool:
        SQL = """
        UPDATE
            domains
        SET
            deleted_at = ?
        WHERE
            id = ?
        """
        return self.execute(SQL, (datetime.now(), domain.id)) == 1

    def fetch(self, id:int) -> Domain:
        SQL = """
        SELECT
            id,
            description
        FROM
            domains
        WHERE
            id = ? and deleted_at is null
        """

        fetched = self.fetch_one(SQL, (id,))
        if fetched:
            return Domain(id=fetched[0], description=fetched[1])
        else:
            return None


import numpy
from PIL import Image, ImageDraw
import os

def get_image(path):
	for filename in os.listdir(path):
		if 'jpg' in filename or 'jpeg' in filename or 'png' in filename:
			img = Image.open(filename)
			shape = img.size
			img2 = img.crop((10, 10, shape[0], shape[1]*0.7))
			index = filename.find('.')
			newname = filename[0:index] + '_crop' + filename[index:]
			img2.save(os.path.join(path,newname))

def main():
	get_image('/Users/lxb/Documents/yyc/scripts/booksproject/the_tale_of_peter_rabbit/')

if __name__ == '__main__':
	main()


import os.path

from PIL import Image

# 读取原始图像
srcImage = r"E:\code\google-chrome-extensions\project-litongjava\google-chrome-apps-chatgpt\icons\src.png"
dirname = os.path.dirname(srcImage)

# 输出不同尺寸的图像
# Image.open(srcImage).resize((256, 256)).save(os.path.join(dirname, "icon.ico"))
srcImage = Image.open(srcImage)
srcImage.resize((16, 16)).save(os.path.join(dirname, "icon16.png"))
srcImage.resize((48, 48)).save(os.path.join(dirname, "icon48.png"))
srcImage.resize((128, 128)).save(os.path.join(dirname, "icon128.png"))
print("done")


import numpy as np

import torch

def nms(bbox, thresh, score=None, limit=None):
    """Suppress bounding boxes according to their IoUs and confidence scores.
    Args:
        bbox (array): Bounding boxes to be transformed. The shape is
            :math:`(R, 4)`. :math:`R` is the number of bounding boxes.
        thresh (float): Threshold of IoUs.
        score (array): An array of confidences whose shape is :math:`(R,)`.
        limit (int): The upper bound of the number of the output bounding
            boxes. If it is not specified, this method selects as many
            bounding boxes as possible.
    Returns:
        array:
        An array with indices of bounding boxes that are selected. \
        They are sorted by the scores of bounding boxes in descending \
        order. \
        The shape of this array is :math:`(K,)` and its dtype is\
        :obj:`numpy.int32`. Note that :math:`K \\leq R`.

    from: https://github.com/chainer/chainercv
    """

    if len(bbox) == 0:
        return np.zeros((0,), dtype=np.int32)

    if score is not None:
        order = score.argsort()[::-1]
        bbox = bbox[order]
    bbox_area = np.prod(bbox[:, 2:] - bbox[:, :2], axis=1)

    selec = np.zeros(bbox.shape[0], dtype=bool)
    for i, b in enumerate(bbox):
        tl = np.maximum(b[:2], bbox[selec, :2])
        br = np.minimum(b[2:], bbox[selec, 2:])
        area = np.prod(br - tl, axis=1) * (tl < br).all(axis=1)

        iou = area / (bbox_area[i] + bbox_area[selec] - area)
        if (iou >= thresh).any():
            continue

        selec[i] = True
        if limit is not None and np.count_nonzero(selec) >= limit:
            break

    selec = np.where(selec)[0]
    if score is not None:
        selec = order[selec]
    return selec.astype(np.int32)

def postprocess(prediction, num_classes, conf_thre=0.7, nms_thre=0.45):
    """
    Postprocess for the output of YOLO model
    perform box transformation, specify the class for each detection,
    and perform class-wise non-maximum suppression.
    Args:
        prediction (torch tensor): The shape is :math:`(N, B, 4)`.
            :math:`N` is the number of predictions,
            :math:`B` the number of boxes. The last axis consists of
            :math:`xc, yc, w, h` where `xc` and `yc` represent a center
            of a bounding box.
        num_classes (int):
            number of dataset classes.
        conf_thre (float):
            confidence threshold ranging from 0 to 1,
            which is defined in the config file.
        nms_thre (float):
            IoU threshold of non-max suppression ranging from 0 to 1.

    Returns:
        output (list of torch tensor):

    """
    box_corner = prediction.new(prediction.shape)
    box_corner[:, :, 0] = prediction[:, :, 0] - prediction[:, :, 2] / 2
    box_corner[:, :, 1] = prediction[:, :, 1] - prediction[:, :, 3] / 2
    box_corner[:, :, 2] = prediction[:, :, 0] + prediction[:, :, 2] / 2
    box_corner[:, :, 3] = prediction[:, :, 1] + prediction[:, :, 3] / 2
    prediction[:, :, :4] = box_corner[:, :, :4]

    output = [None for _ in range(len(prediction))]
    for i, image_pred in enumerate(prediction):
        # Filter out confidence scores below threshold
        # class_pred = torch.max(image_pred[:, 5:5 + num_classes], 1)
        # class_pred = class_pred[0]
        conf_mask = (image_pred[:, 4]  >= conf_thre).squeeze()
        image_pred = image_pred[conf_mask]

        # If none are remaining => process next image
        if not image_pred.size(0):
            continue
        # Get detections with higher confidence scores than the threshold
        ind = (image_pred[:, 5:] * image_pred[:, 4][:, None] >= conf_thre).nonzero()
        # Detections ordered as (x1, y1, x2, y2, obj_conf, class_conf, class_pred)
        detections = torch.cat((
                image_pred[ind[:, 0], :5],
                image_pred[ind[:, 0], 5 + ind[:, 1]].unsqueeze(1),
                ind[:, 1].float().unsqueeze(1)
                ), 1)
        # Iterate through all predicted classes
        unique_labels = detections[:, -1].cpu().unique()
        if prediction.is_cuda:
            unique_labels = unique_labels.cuda()
        for c in unique_labels:
            # Get the detections with the particular class
            detections_class = detections[detections[:, -1] == c]
            nms_in = detections_class.cpu().numpy()
            nms_out_index = nms(
                nms_in[:, :4], nms_thre, score=nms_in[:, 4]*nms_in[:, 5])
            detections_class = detections_class[nms_out_index]
            if output[i] is None:
                output[i] = detections_class
            else:
                output[i] = torch.cat((output[i], detections_class))

    return output


from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time

driver = webdriver.Chrome('chromedriver.exe')
driver.maximize_window()
URL ='https://bdjobs.com'

driver.get(URL)
time.sleep(7)

url_elements = driver.find_elements_by_xpath("//a[@href]")
url_elements_list = []
for url_element in url_elements:
    url_elements_list.append(url_element.get_attribute("href"))
    print(url_elements_list)

driver.close()


from selenium import webdriver
from selenium.webdriver.support.ui import Select
import pprint as pp
import pandas as pd

def scrape_table(table):
	table_rows = table.find_elements_by_tag_name('tr')
	data = [scrape_row(row) for row in table_rows]
	return(data)

def scrape_row(row):
	attrs = row.find_elements_by_tag_name('td')
	data = [attr.text for attr in attrs]
	return(data)

def find_symbol(team_name):
	#(team_name)
	indx = symbols['teamName'].str.contains(team_name)
	print('{} :::: {}'.format(team_name,symbols.loc[indx,'teamName']))
	try:
		return(symbols.loc[indx,'crestURI'].tolist()[0])
	except:
		return('Noneee')

symbols = pd.read_csv('symbols.csv',header=0)

driver = webdriver.Chrome()

# Pulling

LEAGUES = dict(zip(['laliga','serie-a','ligue-1','premier-league','bundesliga'],
				   ['La Liga','Serie A','Ligue 1','Premier League','Bundesliga']))
fin = []
for lg in LEAGUES.keys():
	driver.get('http://www.beinsports.com/us/{}/statistics'.format(lg))
	form = driver.find_elements_by_css_selector("form.ranking-category")[1]
	select = Select(form.find_element_by_tag_name('select'))
	select.select_by_visible_text('Assists')
	# Desired Stats
	pandas_header = ['Pos','Name','GP','Team','Stat']
	STATS = ['goals','assists','chances_created']
	STATS = dict(zip(STATS,['Goals','Assists','Ch. Created']))
	for st in STATS.keys():
		select.select_by_value(st)
		table = driver.find_element_by_css_selector('div.'+st)
		dff = pd.DataFrame(scrape_table(table)[1:],columns=pandas_header)
		dff['Statname'] = STATS[st]
		dff['League'] = LEAGUES[lg]
		dff['ABBR'] = 'FCB'
		dff['CREST'] = 'http://upload.wikimedia.org/wikipedia/de/d/de/Getafe_CF.svg'
		dff['Name'] = dff['Name'].str.split().str[-1]
		fin.append(dff)
datas = pd.concat(fin)
driver.close()

#datas = pd.read_csv('super_table5.csv',header=0)

## Applying Crests
datas['CREST'] = datas['Team'].apply(find_symbol)

## Reseting index
datas.reset_index(inplace=True,drop=True)

# Saving
datas.to_csv('super_table5.csv',index=False)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
@author: jacobstachowicz
"""
import numpy as np
import numpy.matlib as npm
import pickle as p
import random
import os
import matplotlib.pyplot as plt
import math

directory = r"C:\[insert path]\code\cifar-10-batches-py" # for windows
#directory = "/[insert path]/code/cifar-10-batches-py"   # for mac/linux

def paramChange(params, new, i):
    stuff    = [np.copy(i) for i in params]
    stuff[i] = new
    return stuff

def computeNumGrads(X, Y, params, gamma, beta, lmbda, val, batchNorm):
    grads = [np.copy(i) for i in params]
    gGrads = [np.copy(i) for i in gamma]
    bGrads = [np.copy(i) for i in beta]
    c     = cost(X, Y, params, gamma, beta, lmbda, batchNorm)

    for i in range(int(len(params)/2)):
        W = params[i]
        for k in range(np.size(W, 0)):
            for j in range(W.shape[1]):
                Wi = np.array(W)
                Wi[k,j] += val
                cj = cost(X, Y, paramChange(params, Wi, i), gamma, beta, lmbda,batchNorm)
                grads[i][k,j] = (cj - c) / val

    for i in range(int(len(grads)/2),len(params)):
        b = params[i]
        for k in range(np.size(b)):
            bi = np.array(b)
            bi[k] += val
            ck = cost(X, Y, paramChange(params, bi, i), gamma, beta, lmbda, batchNorm)
            grads[i][k] = (ck - c) / val

    if(batchNorm):

        for i in range(len(gamma)):
            W = gamma[i]
            for k in range(np.size(W, 0)):
                for j in range(W.shape[1]):
                    Wi = np.array(W)
                    Wi[k,j] += val
                    cj = cost(X, Y, params, paramChange(gamma, Wi, i), beta, lmbda, batchNorm)
                    gGrads[i][k,j] = (cj - c) / val

        for i in range(len(bGrads)):
            b = bGrads[i]
            for k in range(np.size(b)):
                bi = np.array(b)
                bi[k] += val
                ck = cost(X, Y, params, gamma, paramChange(beta, bi, i), lmbda,batchNorm)
                bGrads[i][k] = (ck - c) / val

    return grads, gGrads, bGrads

def plotLoss(cost_train, cost_valid, yLabel):

    eta_min, eta_max, n_s, t_end, cycles = etas

    plt.figure(dpi=120)
    plt.plot(cost_train, "-g", label="training "+ str(yLabel))
    plt.plot(cost_valid, "-r", label="validation" + str(yLabel))
    plt.ylabel(yLabel)
    plt.xlabel('update step')
    plt.legend(loc="lower right")
    plt.xlim(xmin=0)
    plt.ylim(ymin=0)
    plt.grid(True)

    max_x       = int(math.ceil(t_end / 1000.0)) * 1000
    perCycle    = 10
    cycle_len   = n_s*2
    step        = int((cycle_len)/perCycle)
    array_len   = len(cost_valid)
    my_ticks    = np.arange(0, max_x, step)
    frequency   = 10

    plt.xticks(np.arange(0, array_len+1, frequency), my_ticks[::frequency])
    plt.show()

def unpickle(file):
    with open(file, 'rb') as fo:
        dict = p.load(fo, encoding='bytes')
    return dict

def fetch_batches(type_of_batch):
    arr = []
    path = os.fsencode(directory)
    for file in os.listdir(path):
        filename = os.fsdecode(file)
        if filename.startswith(type_of_batch):
            arr.append(unpickle("cifar-10-batches-py/" + filename))
    return arr

def largestBatch():
    data = fetch_batches("data_batch")
    X, Y, y    = loadBatch(data[0])
    validSize = 10
    for i in range(1, len(data)):
        newX, newY, newy    = loadBatch(data[i])
        X = np.concatenate((X, newX), axis=0)
        Y = np.concatenate((Y, newY), axis=0)
        y = np.concatenate((y, newy), axis=0)

    Xtrain = np.delete(X, np.s_[len(X)-validSize:len(X)], axis=0)
    Ytrain = np.delete(Y, np.s_[len(Y)-validSize:len(Y)], axis=0)
    ytrain = y[0:len(y)-validSize]
    Xvalid = np.delete(X, np.s_[0:len(X)-validSize], axis=0)
    Yvalid  = np.delete(Y, np.s_[0:len(Y)-validSize], axis=0)
    yvalid  = y[len(y)-validSize:len(y)]
    return Xtrain, Ytrain, ytrain, Xvalid , Yvalid , yvalid

def emptyLists(amount):
    arr = []
    for i in range(amount):
        arr.append([])
    return arr

def fixedLists(amount, size):
    arr = []
    for i in range(amount):
        arr.append([None] * size)
    return arr

def hot_ones(labels):
    hot_list = np.zeros( (len(labels), 10))
    for i in range(len(labels)):
        hot_list[i][labels[i]] = 1
    return hot_list

def loadBatch(batch):
    X = np.asarray(batch[b'data'])
    y = np.asarray(batch[b'labels'])
    Y = hot_ones(y)
    return X, Y, y

def multiT(X,Y,Xv,Yv):
    return X.T, Y.T, Xv.T, Yv.T

def createMiniBatch(X, Y, y, size):
    newX = np.zeros([len(X[:,0]), size])
    newY = np.zeros( (len(Y), size))
    newy = np.zeros([size])
    a = np.arange(len(y))
    random.shuffle(a)

    for i in range(size):
        newX[:,i] = X[:,a[i]]
        newY[:,i] = Y[:,a[i]]
        newy[i] = y[a[i]]

    return newX, newY, newy

def createMiniBatches(X, Y, y, n_batches):
    size = int(np.size(X, 1)/n_batches)
    batch_array = [None] * n_batches
    a = np.arange(np.size(X, 1))
    random.shuffle(a)
    for ii in range(int(n_batches)):
        step = ii * size
        newY = np.zeros((np.size(Y, 0), size))
        newX = np.zeros([np.size(X, 0), size])
        newy = np.zeros([size])
        for i in range(size):
            newX[:,i] = X[:,a[i+step]]
            newY[:,i] = Y[:,a[i+step]]
            newy[i]   = y[a[i+step]]
        b_dict =	{
            "X": newX,
            "Y": newY,
            "y": newy
        }
        batch_array[ii] = b_dict
    return batch_array

def unpackMiniBatch(b, i):
    return b[i]["X"], b[i]["Y"], b[i]["y"]

def preprocess(X):
    X = X - npm.repmat(np.mean(X, axis = 0), len(X), 1)
    return np.divide(X, npm.repmat(np.std(X, axis = 0), len(X), 1))

def createParams(info):
    Ws = []
    bs = []
    for i in range(len(info)-1):
        Ws.append(np.random.randn(info[i+1], info[i]) * np.sqrt(2/info[i] ) )
        bs.append(np.zeros((info[i+1], 1)))
    return Ws + bs

def createParamsSensitive(info, sig):
    Ws = []
    bs = []
    for i in range(len(info)-1):

        Ws.append(np.random.normal(0, sig, (info[i+1], info[i]) ) )
        bs.append(np.zeros((info[i+1], 1)))
    return Ws + bs

def createYB(info):
    Y = []
    B = []
    for i in range(len(info)-1):
        Y.append(np.ones([info[i+1], 1]))
        B.append(np.zeros([info[i+1], 1]))

    return Y, B

def leakyReLu(s):
    return np.maximum(s*0.1, 0)

def reLu(s):
    return np.maximum(s, 0)

def softMax(s):
    return np.exp(s) / np.sum(np.exp(s), axis = 0)

def batchNormalise(S, mu, v):
    eps = 1e-9
    return ( S - mu ) / np.sqrt(eps + v)

def forward(*args):
    mu, v, x, s, s_hat = emptyLists(5)
    X, params, gamma, beta, batchNorm = args[:5]
    testing = len(args) == 7
    if(testing):
        mu, v = args[-2:]
    k = int(len(params) / 2)
    x.append(np.copy(X))

    for i in range(k - 1):
        s.append(np.matmul(params[i], x[i]) + params[k + i])        #5
        if(batchNorm):
            if(not testing):
                mu.append(r(np.mean(s[i], axis = 1)))
                v.append( r(np.var(s[i],  axis = 1)))

            s_hat.append(batchNormalise(s[i], mu[i], v[i]))         #6
            s_tmp = np.multiply(gamma[i], s_hat[i]) + beta[i]       #7
            x.append(reLu(s_tmp))                                   #8
        else:
            x.append(reLu(s[i]))

    s_last = np.matmul(params[k-1], x[k-1]) + params[2*k -1]
    p  = softMax(s_last)
    return p, x, s, s_hat, mu, v

def crossEntropyLoss(P, Y):
    lcrosssum = 0;
    for i in range(np.size(Y,1)):
        lcrosssum -= np.log(np.dot(Y[:,i], P[:,i]))
    return lcrosssum

def regularization(params, lmbda):
    return lmbda *  np.sum([np.sum(params[i] * params[i]) for i in range(int(len(params)/2))])

def cost(X, Y, weightsAndBias, gamma, beta, lmbda, batchNorm):
    f = forward(X, weightsAndBias, gamma, beta, batchNorm)
    lcrosssum = crossEntropyLoss(f[0], Y)
    reg = regularization(weightsAndBias, lmbda)
    return (1/np.size(X, 1)) * lcrosssum + reg

def ComputeAcc(X, y, params, gamma, beta, mu, v, batchNorm):
    P = forward(X, params, gamma, beta, batchNorm, mu, v)[0]
    corr = 0
    tot = len(y)
    for i in range(tot):
        corr += 1 if y[i] == np.argmax(P[:,i]) else 0
    return round(100*(corr/tot),2)

def ComputeEnsambleAcc(X, y, nets, batchNorm):
    params, gamma, beta, mu, v = nets[0]

    P = forward(X, params, gamma, beta, batchNorm, mu, v)[0]

    for n in range(len(nets)-1):
        params, gamma, beta, mu, v = nets[n+1]
        Pt = forward(X, params, gamma, beta, batchNorm, mu, v)[0]
        P += Pt

    corr = 0
    tot = len(y)
    for i in range(tot):
        corr += 1 if y[i] == np.argmax(P[:,i]) else 0
    return round(100*(corr/tot),2)

def printAcc(Xo, yo, Xv, yv,params, gamma, beta, mu, v,batchNorm ):
    print("post train acc: "+str(ComputeAcc(Xo, yo, params, gamma, beta, mu, v,batchNorm)) + " %")
    print("post valid acc: "+str(ComputeAcc(Xv, yv, params, gamma, beta, mu, v,batchNorm)) + " %")

def r(n):
    return np.reshape(n, (len(n), 1))

def toParams(W1, W2, b1, b2):
    return W1, W2, b1, b2

def cLearnRates(etas, t):
    eta_min, eta_max, n_s, i, cycles = etas
    l = int(t / (n_s*2))
    if(t % (2*n_s) < n_s):
        return eta_min + ( (t-(2*l * n_s)) /n_s) * (eta_max - eta_min)
    else:
        return eta_max - ( (t-((2*l+1) * n_s)) /n_s) * (eta_max - eta_min)

def batch_norm_back_pass(G, S, mu, v, div):
    eps = 1e-5

    sigma_1 = np.power((v+eps), -0.5) # (1/(np.sqrt(v+eps)))                #31
    sigma_2 = np.power((v+eps), -1.5) #(1/((v * np.sqrt(v))+eps))           #32

    G1 = np.multiply(G, sigma_1)                                            #33
    G2 = np.multiply(G, sigma_2)                                            #34

    D = S - mu                                                              #35
    c = r(np.sum(np.multiply(G2, D), axis = 1))                             #36

    part1 = div * r( np.sum(G1, axis=1))
    part2 = div * np.multiply(D, r(np.sum(c, axis = 1)))

    return  G1 - part1 - part2

def backwards(X, Y, params, g, b, lmbda, batchNorm):
    k = int(len(params) / 2)
    div = (1/np.size(Y,1))

    P, h, s, s_hat, mu, v = forward(X, params, g, b, batchNorm)

    grads = [np.copy(i) for i in params]
    gGrads = np.copy(g)
    bGrads = np.copy(b)

    G = P - Y                                                                           #21
    grads[k-1] = div * np.matmul(G, np.transpose(h[k-1])) + 2 * lmbda * params[k-1]     #22 ekv 1
    grads[2*k-1] = r(div * np.sum(G, axis=1))                                           #22 ekv 2

    G = np.matmul(np.transpose(params[k-1]), G)                                         #23
    G = np.multiply(G, (h[k-1] > 0))                                                    #24

    for i in range(k-2, -1, -1):
        if(batchNorm):
            gGrads[i] = r(div * np.sum(np.multiply(G, s_hat[i]), axis = 1))             #25 ekv 1
            bGrads[i] = r(div * np.sum(G, axis=1))                                      #25 ekv 2

            G = np.multiply(G, r(np.sum(g[i], axis=1)))                                 #26
            G = batch_norm_back_pass(G, s[i], mu[i], v[i], div )                        #27

        grads[i] = div * np.matmul(G, np.transpose(h[i])) + 2 * lmbda * params[i]
        grads[i+k] = r(div * np.sum(G, axis=1))

        if(i > 0):
            G = np.matmul(np.transpose(params[i]), G)
            G = np.multiply(G, (h[i] > 0))

    return grads, gGrads, bGrads, mu, v

def miniBatchGD(G, n_batches, etas, params, gamma, beta, l, batchNorm, printStuff = False):
    Xo, Yo, yo, Xv, Yv, yv = G
    Xt, Yt, yt             = test

    eta_min, eta_max, n_s, t_end, cycles = etas

    stepPrint = int(2*n_s)/10
    max_x     = (int(math.ceil(t_end / 1000.0)) * 1000)
    size      = int(max_x/stepPrint)+1

    cost_t, cost_v, loss_t, loss_v, acc_t, acc_v, t_rate = fixedLists(7, size)
    batches = createMiniBatches(Xo, Yo, yo, n_batches)
    ii = 0
    mu_avg = 0.0
    v_avg  = 0.0
    alpha = 0.9
    for t in range(t_end):

        eta = cLearnRates(etas, t)
        if(t % n_batches == 0):
            batches = createMiniBatches(Xo, Yo, yo, n_batches)
        i = t % n_batches
        Xb, Yb, yb = unpackMiniBatch(batches, i)

        grads, gGrads, bGrads, mu, v = backwards(Xb, Yb, params, gamma, beta, l, batchNorm)

        for i in range(len(params)):
            params[i] = params[i] - eta * grads[i]

        if(batchNorm):
            for i in range(len(gamma)):
                gamma[i] = gamma[i] - eta * gGrads[i]
                beta[i]  = beta[i]  - eta * bGrads[i]
            if(t ==0):
                mu_avg = mu
                v_avg = v
            for i in range(len(mu_avg)):
                mu_avg[i] = mu_avg[i]*alpha + (1.0-alpha)*mu[i]
                v_avg[i] = v_avg[i]*alpha + (1.0-alpha)*v[i]

        if(t% stepPrint==0 or t == t_end-1):
            if((t% (stepPrint*4)==0) and printStuff):
                print(str(100 * (t/t_end)) + " Procent done")
            cost_t[ii]  = cost(Xo, Yo, params, gamma, beta, l, batchNorm)
            cost_v[ii]  = cost(Xv, Yv, params,gamma, beta, l, batchNorm)

            loss_t[ii] = cost(Xo, Yo, params, gamma, beta,0, batchNorm)
            loss_v[ii] = cost(Xv, Yv, params,gamma, beta, 0, batchNorm)

            acc_t[ii]   = ComputeAcc(Xo, yo, params, gamma, beta, mu_avg, v_avg, batchNorm)
            acc_v[ii]   = ComputeAcc(Xv, yv, params, gamma, beta, mu_avg, v_avg, batchNorm)
            if(printStuff):
                printAcc(Xo, yo, Xv, yv,params, gamma, beta, mu_avg, v_avg ,batchNorm )
                print("TestACC: " + str(ComputeAcc(Xt, yt, params, gamma, beta, mu_avg, v_avg, batchNorm)))

            t_rate[ii] = eta

            ii += 1
    final_v_acc = ComputeAcc(Xv, yv, params, gamma, beta, mu_avg, v_avg,batchNorm)
    if(printStuff):
        plotLoss(cost_t, cost_v, "Cost")
        plotLoss(loss_t, loss_v, "Loss")
        plotLoss(acc_t, acc_v, "Accuracy")
        plotLoss(t_rate, t_rate, "ting rate")

        printAcc(Xo, yo, Xv, yv,params, gamma, beta, mu_avg, v_avg, batchNorm )
        Xt, Yt, yt = test
        print("test acc: " + str(ComputeAcc(Xt, yt, params, gamma, beta, mu_avg, v_avg,batchNorm)))
    return params, gamma, beta,mu_avg, v_avg, final_v_acc

def logLambda():
    e_min = -4
    e_max = -1.5
    l = e_min + (e_max - e_min) * random.random()
    return np.power(10, l);

def logLambdaOrdered(i):
    e_min = -2.5
    e_max = -2.15 #-1.5
    l = e_min + (e_max - e_min) * i
    return np.power(10, l);

def bestParams(G, n_batches, etas, weightsAndBias, gamma, beta, l, test, batchNorm, times):

    eta_min, eta_max, n_s, t_end, cycles = etas
    testX, testY, testy = test

    final_valid_acc = 0
    second_valid_acc = 0

    final_lmbda = lmbda
    s_final_lmbda = final_lmbda

    final_params = [np.copy(i) for i in weightsAndBias]
    final_gamma = np.copy(gamma)
    final_beta = np.copy(beta)
    final_mu = None
    final_v = None

    s_final_params = [np.copy(i) for i in weightsAndBias]
    s_final_gamma = np.copy(gamma)
    s_final_beta = np.copy(beta)
    s_final_mu = None
    s_final_v = None

    for i in range(times):
        new_lmbda = logLambda()
        print(i/times)
        print("search nr: " + str(i+1) + " with lambda: "  + str(new_lmbda))
        p = [np.copy(i) for i in weightsAndBias]
        g = np.copy(gamma)
        b = np.copy(beta)
        new_params, new_gamma, new_beta,mu,v, v_acc = miniBatchGD(G, n_batches, etas, p ,g, b,new_lmbda, batchNorm)
        print("post vali acc:  " + str(v_acc))
        print("post test acc:  " + str(ComputeAcc(testX, testy, new_params, new_gamma, new_beta, mu, v, batchNorm)))
        if(v_acc > final_valid_acc):
            s_final_lmbda = final_lmbda
            s_final_params = final_params
            s_final_gamma = final_gamma
            s_final_beta = final_beta
            s_final_mu = final_mu
            s_final_v = v
            second_valid_acc = final_valid_acc

            final_lmbda = new_lmbda
            final_params = new_params
            final_gamma = new_gamma
            final_beta = new_beta
            final_mu = mu
            final_v = v
            final_valid_acc = v_acc
        ComputeAcc(testX, testy, new_params, new_gamma, new_beta, mu, v, batchNorm)
    test_acc =ComputeAcc(testX, testy, final_params, final_gamma,final_beta, final_mu, final_v, batchNorm)
    s_test_acc =ComputeAcc(testX, testy, s_final_params, s_final_gamma,s_final_beta, s_final_mu, s_final_v, batchNorm)
    print("##################")
    print("best lambda : " + str(final_lmbda))
    print("best valid acc: " + str(final_valid_acc))
    print("gave test acc: " + str(test_acc))
    print("------------------")
    print("2nd best lambda : " + str(s_final_lmbda))
    print("gave valid acc: " + str(second_valid_acc))
    print("gave test acc: " + str(s_test_acc))

def gradNorm(aGrad,nGrad,name):
    print("Norm_" + name + ": " + str(np.linalg.norm(aGrad - nGrad) ))

def relativeError(aGrad,nGrad, name):
    eps   = 1e-05
    upper = np.linalg.norm(aGrad - nGrad)
    lower = max(eps, np.linalg.norm(aGrad) + np.linalg.norm(nGrad))
    print("relative error for " + name + ": " + str(upper/lower))

def testGradients(X, Y, y, batchsize, weightsAndBias, gamma, beta,lmbda,batchNorm):
    miniX, miniY, miniy                    = createMiniBatch(X, Y, y, batchsize)

    aGrads, gGrads, bGrads, mu, v = backwards(miniX, miniY, weightsAndBias,gamma,beta, lmbda, batchNorm)
    nGrads, ngGrads, nbGrads = computeNumGrads(miniX, miniY, weightsAndBias, gamma, beta, lmbda, 1e-5,batchNorm)

    netLen = len(weightsAndBias)
    lenG = len(gGrads)
    for i in range(netLen):
        name = "W"+str(i+1)
        if i >= int(netLen/2):
            name = "b"+str(i+1 - int(netLen/2))
        relativeError(aGrads[i],nGrads[i],name)

    if(batchNorm):
        for i in range(lenG):
            relativeError(gGrads[i],ngGrads[i],"Gamma")
            relativeError(bGrads[i],nbGrads[i],"Beta")

################################  Exercise 3 Bonus   ################################

for i in range(1):

    testX, testY, testy    = loadBatch(unpickle(directory + "/test_batch"))

    trainX, trainY, trainy, validX, validY, vaildy= largestBatch() # training set of size 49000 and test set of size 1000
    trainX = preprocess(trainX)
    validX = preprocess(validX)
    testX  = preprocess(testX)

    """ Used for reducing dimensionality (use only when testing gradients)"""
#    trainX = np.delete(trainX, np.s_[10:len(trainX)], axis=1)

    trainX, trainY, validX, validY  = multiT(trainX, trainY, validX, validY)
    test = testX.T, testY.T, testy
    Xt, Yt, yt = test

    val       = 1e-5
    k         = np.size(trainY, 0)
    d         = np.size(trainX, 0)
    n         = np.size(trainX, 1)
    m         = 10

    n_batches = 450
    n_b_size  = 100
    lmbda     = 0.0044668359215096305
    l_mod     = 0.0002

    cycles    = 4
    eta_min   = 1e-5
    eta_max   = 1e-1
    n_s       = int(5 * int(n / n_b_size))
    t         = cycles*2*n_s
    etas      = eta_min, eta_max, n_s, t, cycles

    networkDim = [d, 50, 30, 20, 20, 10, 10, 10, 10, k]
    """ Use only when testing sensitivity to initialisation"""
#    weightsAndBias = createParamsSensitive(networkDim, 1e-1)
    weightsAndBias = createParams(networkDim)
    Gamma, Beta =           createYB(networkDim)
    GG                      = trainX, trainY, trainy, validX, validY, vaildy

    batchNorm = True
    printStuff = True

    """ For testing gradients, dont forget to reduce dimensionality on row 591"""
#    testGradients(trainX, trainY, trainy, 1000, weightsAndBias, Gamma, Beta, lmbda, batchNorm)

    """ Mini batchGD"""
 #   params, gamma, beta, mu, v, finalAcc   = miniBatchGD(GG, n_batches, etas, weightsAndBias, Gamma, Beta, lmbda, batchNorm, printStuff)

    """ For finding best lambda: """
#    bestParams(GG, n_batches, etas, weightsAndBias, Gamma, Beta, lmbda, test, batchNorm, 20) # for finding best lambda

    ensamble = True
    if(ensamble):

        netA = [d, 2000, 1000, 2000, k]
        netB = [d, 2000, 1000, 2000, k]
        netC = [d, 500, 1000, 1000, 500, k]
        netD = [d, 1000, 4000, 1000, k]
        netE = [d, 3000, 3000, k]

        weightsAndBiasA     = createParams(netA)
        weightsAndBiasB     = createParams(netB)
        weightsAndBiasC     = createParams(netC)
        weightsAndBiasD     = createParams(netD)
        weightsAndBiasE     = createParams(netE)

        AGamma, ABeta =           createYB(netA)
        BGamma, BBeta =           createYB(netB)
        CGamma, CBeta =           createYB(netC)
        DGamma, DBeta =           createYB(netD)
        EGamma, EBeta =           createYB(netE)

        paramsC, gammaC, betaC, muC, vC, finalAccC   = miniBatchGD(GG, n_batches, etas, weightsAndBiasC, CGamma, CBeta, lmbda, batchNorm, printStuff)
        paramsA, gammaA, betaA, muA, vA, finalAccA   = miniBatchGD(GG, n_batches, etas, weightsAndBiasA, AGamma, ABeta, lmbda, batchNorm, printStuff)
        paramsB, gammaB, betaB, muB, vB, finalAccB   = miniBatchGD(GG, n_batches, etas, weightsAndBiasB, BGamma, BBeta, 0.03, batchNorm, printStuff)
        paramsD, gammaD, betaD, muD, vD, finalAccD   = miniBatchGD(GG, n_batches, etas, weightsAndBiasD, DGamma, DBeta, lmbda+0.0002, batchNorm, printStuff)
        paramsE, gammaE, betaE, muE, vE, finalAccE   = miniBatchGD(GG, n_batches, etas, weightsAndBiasE, EGamma, EBeta, lmbda-0.0002, batchNorm, printStuff)

        print("######## i is: "+ str(i) +" ##########")
        print("individual test accuracy: ")
        print( str( ComputeAcc(Xt, yt, paramsA, gammaA, betaA, muA, vA, batchNorm)))
        print( str( ComputeAcc(Xt, yt, paramsB, gammaB, betaB, muB, vB, batchNorm)))
        print( str( ComputeAcc(Xt, yt, paramsC, gammaC, betaC, muC, vC, batchNorm)))
        print( str( ComputeAcc(Xt, yt, paramsD, gammaD, betaD, muD, vD, batchNorm)))
        print( str( ComputeAcc(Xt, yt, paramsE, gammaE, betaE, muE, vE, batchNorm)))

        print("---------------------------")
        netsA = paramsA, gammaA, betaA, muA, vA
        netsB = paramsB, gammaB, betaB, muB, vB
        netsC = paramsC, gammaC, betaC, muC, vC
        netsD = paramsD, gammaD, betaD, muD, vD
        netsE = paramsE, gammaE, betaE, muE, vE

        wholeTeam           = netsA, netsB, netsC, netsD, netsE
        outer               = netsA, netsE
        middleAndOuter      = netsA, netsC, netsE
        vampire             = netsB, netsD
        middleThree         = netsB, netsC, netsD
        allButMiddle        = netsA, netsB, netsD, netsE

        acc_wholeTeam       = ComputeEnsambleAcc(Xt, yt, wholeTeam, batchNorm)
        acc_outer           = ComputeEnsambleAcc(Xt, yt, outer, batchNorm)
        acc_middleAndOuter  = ComputeEnsambleAcc(Xt, yt, middleAndOuter, batchNorm)
        acc_vampire         = ComputeEnsambleAcc(Xt, yt, vampire, batchNorm)
        acc_middleThree     = ComputeEnsambleAcc(Xt, yt, middleThree, batchNorm)
        acc_allButMiddle    = ComputeEnsambleAcc(Xt, yt, allButMiddle, batchNorm)

        print("wholeTeam aensemble accuracy:     " + str(acc_wholeTeam))
        print("outer ensemble accuracy:          " + str(acc_outer))
        print("middleAndOuter ensemble accuracy: " + str(acc_middleAndOuter))
        print("vampire ensemble accuracy:        " + str(acc_vampire))
        print("middleThree ensemble accuracy:    " + str(acc_middleThree))
        print("allButMiddle ensemble accuracy:   " + str(acc_allButMiddle))
        print("##################################")


from aiogram import types
from aiogram.dispatcher import FSMContext

from app import bot, dp
from database import TelegramChannel
from states import CreatedGivesStates
from keyboards import *

@dp.message_handler(
    state=CreatedGivesStates.add_channel,
)
async def get_channel_data(
    jam: types.Message,
    state: FSMContext
):
    try:

        channel_data = jam.forward_from_chat

        if channel_data.type == 'channel':

            try:
                member_info = await bot.get_chat_member(
                    chat_id=channel_data.id,
                    user_id=bot.id
                )

                if member_info.status == 'administrator':
                    model = TelegramChannel()

                    if not await model.exists_channel(
                            channel_id=channel_data.id
                    ):

                        state_data = await state.get_data()

                        await model.add_channel(
                            owner_id=jam.from_user.id,
                            channel_id=channel_data.id,
                            give_callback_value=state_data['give_callback_value'],
                            name=channel_data.title
                        )

                        await jam.answer(
                            '✅  Канал успешно добавлен',
                            reply_markup=kb_admin_manage_channels
                        )
                        await CreatedGivesStates.manage_channels.set()

                    else:
                        await jam.answer(
                            '🚫  Данный канал уже добавлен, попробуйте другой: ',
                            reply_markup=kb_admin_cancel_action
                        )

                else:
                    await jam.answer(
                        '🚫  Ошибка! Проверьте права бота и перешлите репостом сообщение из канала еще раз: ',
                        reply_markup=kb_admin_cancel_action
                    )

            except Exception as error:
                print(error)
                await jam.answer(
                    '🚫  Ошибка! Проверьте права бота и перешлите репостом сообщение из канала еще раз: ',
                    reply_markup=kb_admin_cancel_action
                )

        else:
            await jam.answer(
                '🚫  Это не канал, попробуйте еще раз:',
                reply_markup=kb_admin_cancel_action
            )

    except AttributeError:
        await jam.answer(
            '🚫  Это не канал, попробуйте еще раз:',
            reply_markup=kb_admin_cancel_action
        )


#!/usr/bin/env python
"""Simple A* Example script

Filename: search.py
Author: James Casey
Date Created: 2019-12-02
Last Updated: 2019-12-03
"""

import click
import logging
import structlog
from structlog.stdlib import LoggerFactory
import pygame
import random
from enum import Enum
from pdb import set_trace

structlog.configure(logger_factory=LoggerFactory())
log = structlog.get_logger()

debug = log.debug
info = log.info
warning = log.warning
error = log.error

# Global stuff to make life easier
BLACK = (0, 0, 0)
WHITE = (255, 255, 255)
RED = (255, 0, 0)
GREEN = (0, 255, 0)
BLUE = (0, 0, 255)
CYAN = (0, 255, 255)
YELLOW = (255,255,0)

WIDTH=800
HEIGHT=600
SQUARE_SIZE = 40
MAX_X = int(WIDTH / SQUARE_SIZE) - 1
MAX_Y = int(HEIGHT / SQUARE_SIZE) - 1

def square_center(square):
    """ Get the center of a given square. """

    x = square[0]*SQUARE_SIZE + 1 + SQUARE_SIZE / 2
    y = square[1]*SQUARE_SIZE + 1 + SQUARE_SIZE / 2

    return (x,y)

def get_neighbors(square, clicked_squares=[], plus_only=False):
    """ Get a list of the neighbors for a square. """

    neighbors = []

    # Loop over the possible x positions
    for i in [square[0] - 1, square[0], square[0] + 1]:

        # make sure we stay in our region
        if i < 0 or i > MAX_X:
            continue

        # Loop over the possible y positions
        for j in [square[1] - 1, square[1], square[1] + 1]:

            # make sure we stay in our region
            if j < 0 or j > MAX_Y:
                continue

            # skip the given square
            if i == square[0] and j == square[1]:
                continue

            # Include/Exclude the diagonal paths
            if plus_only is True:
                if  i == square[0] - 1 and j !=  square[1]:
                    continue
                if  i == square[0] + 1 and j !=  square[1]:
                    continue

            # Deal with barriers
            if [i,j] in clicked_squares:
                continue

            neighbors.append([i,j])

    return neighbors

class PathNode:
    """ Store the path square information. """

    def __init__(self, parent=None, pos=None, start_square=None, end_square=None):

        self.parent_node = parent
        self.pos = pos
        self.start_square = start_square
        self.end_square = end_square

        self.f = 0
        self.g = 0
        self.h = 0

        self.set_h()

    def set_h(self):
        """ Simple h generator. """

        self.h = ((self.end_square[0] - self.pos[0])
                  + (self.end_square[1] - self.pos[1]))

    def __repr__(self):
        return f"<PathNode: {self.pos}>"

def find_path(start_square, end_square, clicked_squares, plus_only=False):

    # initial set up
    open_list = []
    closed_list = []

    open_list.append(PathNode(parent=None,
                              pos=start_square,
                              start_square=start_square,
                              end_square=end_square))

    # Run until we've found the path
    searching = True
    while searching:

        # Stop when we get have explored all paths
        if len(open_list) == 0:
            break

        # Sort the path nodes to find the shortest so far
        open_list = sorted(open_list, key=lambda entry: -entry.g)
        q = open_list.pop()

        # loop over all the neighbors and update the path information
        for neighbor in get_neighbors(q.pos, clicked_squares, plus_only=plus_only):

            node = PathNode(parent=q,
                            pos = neighbor,
                            start_square=start_square,
                            end_square=end_square)
            node.g = q.g + 1
            node.f = node.g + node.h

            # Stop searching when we get to the end
            if neighbor == end_square:
                searching = False
                break

            # check to see if we already have the node in the open list with
            # a better f value
            check_open = False
            for open_node in open_list:
                if node.pos == open_node.pos and open_node.f <= node.f:
                    check_open = True
                    break
            if check_open is True:
                continue

            # check to see if we already have the node in the closed list with
            # a better f value
            check_closed = False
            for closed_node in closed_list:
                if node.pos == closed_node.pos and closed_node.f <= node.f:
                    check_closed = True
                    continue
            if check_closed is True:
                continue

            # if we make it this far, add the neighbor node to the open list
            open_list.append(node)

        # add the node to the closed list
        closed_list.append(q)

    # open_list = sorted(open_list, key=lambda entry: -entry.g)
    last = node

    # build the path backwards from finish to start
    best_path = []
    while True:
        if last.parent_node is None:
            break
        best_path.append(last.pos)
        last = last.parent_node
    return best_path

@click.command()
@click.option("-n", "--draw-neighbors", is_flag=True, default=False,
                help="Draw neighbors on click")
@click.option("-p", "--plus-only", is_flag=True, default=False,
                help="Set neighbors to only up/down and left/right")
@click.option("-v", "--verbose", is_flag=True, default=False,
              help="Show debuggging information")
def main(verbose, plus_only, draw_neighbors):

    if verbose is True:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.WARNING)

    # initialize everything
    pygame.init()
    pygame.font.init()

    gDisplay = pygame.display.set_mode((WIDTH,HEIGHT))
    clock = pygame.time.Clock()

    start_square = [1,1]
    end_square = [MAX_X-1,MAX_Y-1]
    clicked_squares = []
    path_squares = []
    neighbors = []

    running = True
    while running:

        # Check for interaction
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                running = False

            # Mouse interactions here
            if event.type == pygame.MOUSEBUTTONDOWN:

                # Add/Remove a barrier
                if pygame.mouse.get_pressed() == (1,0,0):
                    pos = pygame.mouse.get_pos()
                    x = pos[0]
                    y = pos[1]
                    new_square = [int(x/SQUARE_SIZE),int(y/SQUARE_SIZE)]

                    # Draw neighbors of the clicked square as well
                    # this is mostly for testing purposes
                    if draw_neighbors is True:
                        neighbors = get_neighbors(new_square,
                                plus_only=plus_only)

                    # Don't do anything to the start/end squares
                    if new_square == start_square or new_square == end_square:
                        continue

                    # toggle the barrier on or off
                    if new_square in clicked_squares:
                        clicked_squares.remove(new_square)
                    else:
                        clicked_squares.append(new_square)

                # Run the search
                elif pygame.mouse.get_pressed() == (0,0,1):
                    path_squares = find_path(start_square, end_square,
                            clicked_squares, plus_only)
                    neighbors = []

        gDisplay.fill(BLACK)

        # Draw the grid
        for i in range(1,int(WIDTH/SQUARE_SIZE)):
            x = SQUARE_SIZE*i
            pygame.draw.line(gDisplay, WHITE, (x,0), (x,HEIGHT),1)

        for i in range(1,int(HEIGHT/SQUARE_SIZE)):
            y = SQUARE_SIZE*i
            pygame.draw.line(gDisplay, WHITE, (0,y), (WIDTH,y),1)

        # Draw best path
        if path_squares is not None:
            last_square = start_square
            for square in path_squares[::-1]:
                x = square[0]*SQUARE_SIZE + 1
                y = square[1]*SQUARE_SIZE + 1
                pygame.draw.rect(gDisplay, YELLOW,
                        pygame.Rect(x,y,SQUARE_SIZE - 1, SQUARE_SIZE - 1))
                pygame.draw.line(gDisplay, RED,
                        (square_center(last_square)),
                        (square_center(square)), 2)
                last_square = square

        # Draw neighbors of clicked squares (if enabled)
        for square in neighbors:
            x = square[0]*SQUARE_SIZE + 1
            y = square[1]*SQUARE_SIZE + 1
            pygame.draw.rect(gDisplay, CYAN, pygame.Rect(x,y,SQUARE_SIZE - 1,
                SQUARE_SIZE - 1))

        # Draw clicked squares
        for square in clicked_squares:
            x = square[0]*SQUARE_SIZE + 1
            y = square[1]*SQUARE_SIZE + 1
            pygame.draw.rect(gDisplay, BLUE, pygame.Rect(x,y,SQUARE_SIZE - 1,
                SQUARE_SIZE - 1))

        # Draw Start Square
        x = start_square[0]*SQUARE_SIZE + 1
        y = start_square[1]*SQUARE_SIZE + 1
        pygame.draw.rect(gDisplay, GREEN, pygame.Rect(x,y,SQUARE_SIZE - 1,
            SQUARE_SIZE - 1))

        # Draw End Square
        x = end_square[0]*SQUARE_SIZE + 1
        y = end_square[1]*SQUARE_SIZE + 1
        pygame.draw.rect(gDisplay, RED, pygame.Rect(x,y,SQUARE_SIZE - 1,
            SQUARE_SIZE - 1))

        pygame.display.update()
        clock.tick(30)

if __name__ == "__main__":
    main()


from datetime import datetime

import albumentations as A
import pytorch_lightning as pl
import torch
from albumentations.pytorch import ToTensorV2
from clearml import Task
from matplotlib import pyplot as plt
from pytorch_lightning.callbacks import ModelCheckpoint, RichProgressBar
from pytorch_lightning.callbacks.progress.rich_progress import RichProgressBarTheme
from simple_parsing import ArgumentParser
from torch.utils.data import Subset

from config import config
from config.args import Args
from config.config import logger
from dataloader import *
from model import Classifier

# Preprocessing function
def get_transform(dataset):

    resize = A.Resize(224, 224)
    normalize = A.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))
    bright_contrast = A.RandomBrightnessContrast(
        brightness_limit=0.1, contrast_limit=0.1, p=0.2
    )
    rgb_shift = A.RGBShift(p=0.05)
    hue_sat = A.HueSaturationValue(p=0.05)
    channel_shuffle = A.ChannelShuffle(p=0.05)
    gamma = A.RandomGamma(p=0.05)
    gray = A.ToGray(p=0.05)
    jitter = A.ColorJitter(p=0.05)
    hor_flip = A.HorizontalFlip(p=0.1)
    rotate = A.Rotate(limit=15, p=0.1)
    crop = A.RandomCrop(height=100, width=100, p=0.1)
    translate = A.Affine(translate_percent=0.1, p=0.1)
    shear = A.Affine(shear=20, p=0.05)
    scale = A.Affine(scale=1.2, p=0.1)
    to_tensor = ToTensorV2()

    if dataset == "train":
        return A.Compose([resize, normalize, to_tensor])
    elif dataset == "val":
        return A.Compose([resize, normalize, to_tensor])

# Dataset function
def prepare_dataset(data_dir):
    try:
        train_dir = str(data_dir) + "/train"
        logger.debug(train_dir)
        val_dir = str(data_dir) + "/val"
        trainset = CustomDataset(data_dir=train_dir, transforms=get_transform("train"))
        valset = CustomDataset(data_dir=val_dir, transforms=get_transform("val"))
        return trainset, valset
    except Exception as e:
        logger.error(f"Got an exception: {e}")

# Sanity check subset function
def create_subset(trainset, valset):

    train_subset = Subset(trainset, range(100))
    val_subset = Subset(valset, range(5))

    return train_subset, val_subset

# Dataloaders function
def create_dataloaders(args, train_subset, val_subset):
    from torch.utils.data import DataLoader

    train_dataloader = DataLoader(
        train_subset, batch_size=args.batch_size, num_workers=24, shuffle=True
    )
    val_dataloader = DataLoader(
        val_subset, batch_size=args.batch_size, num_workers=24, shuffle=False
    )
    return train_dataloader, val_dataloader

def set_device():
    if torch.cuda.is_available():
        device = "gpu"
    else:
        device = "cpu"
    return device

def main():

    device = set_device()
    logger.info(f"Currently using {device} device...")

    # Read args
    logger.info("Reading arguments...")
    parser = ArgumentParser()
    parser.add_arguments(Args, dest="options")
    args_namespace = parser.parse_args()
    args = args_namespace.options

    # Prepare dataset
    logger.info("Preparing datasets...")
    trainset, valset = prepare_dataset(data_dir=config.DATA_DIR)
    logger.info(
        f"""Total training images: {len(trainset)}
Total validation images: {len(valset)}"""
    )

    # Create subset
    # train_subset, val_subset = create_subset(trainset, valset)

    # Create dataloaders
    train_dataloader, val_dataloader = create_dataloaders(args, trainset, valset)

    # Initialize clearml task
    logger.info("Initializing clearml task...")
    task = Task.init(
        project_name="streamlit/image-classification",
        task_name=f"streamlit-image-classification-{datetime.now()}",
    )
    task.connect(args)

    # Saves top-K checkpoints based on "val_loss" metric
    checkpoint_callback = ModelCheckpoint(
        save_top_k=1,
        monitor="loss/val_loss",
        mode="min",
        filename="streamlit-image-classification-{epoch:02d}-{val_loss:.2f}",
    )

    # Create progress bar
    progress_bar = RichProgressBar(
        theme=RichProgressBarTheme(
            description="white",
            progress_bar="#50C878",
            progress_bar_finished="#50C878",
            progress_bar_pulse="#50C878",
            batch_progress="orange",
            time="grey54",
            processing_speed="grey70",
            metrics="white",
        ),
        console_kwargs=None,
        refresh_rate=10,
    )

    # Train
    # configure trainer
    trainer = pl.Trainer.from_argparse_args(
        args,
        callbacks=[checkpoint_callback, progress_bar],
        default_root_dir=config.LOGS_DIR,
        accelerator=device,
        devices=1,
        log_every_n_steps=1,
    )

    # define classifier
    classifier = Classifier()

    # fit the model
    logger.info("Starting training...")
    trainer.fit(classifier, train_dataloader, val_dataloader)

if __name__ == "__main__":
    main()


# 损失函数

import torch.nn as nn
import torch
import math
import torch.nn.functional as F

# log_loss
def log_loss(y, p):
    """
    --- Log_loss computing function
    A function to compute the log loss of a predicted probability p given
    a true target y.

    :param y: True target value
    :param p: Predicted probability
    :return: Log loss.
    """
    p = min(max(p, 10e-15), 1. - 10e-15)
    return -math.log(p) if y == 1 else -math.log(1. - p)

# Cross-Entropy Loss（多分类）用这个 loss 前面不需要加 Softmax 层
def CrossEntropyLoss(predict, target):
    loss = torch.nn.CrossEntropyLoss()(predict, target)
    return loss

# BCELoss(二分类)
def BCELoss(predict, target):
    loss = torch.nn.BCELoss()(predict, target)
    return loss

# MSELoss Mean Square Error
def MSELoss(predict, target):
    loss = torch.nn.MSELoss()(predict, target)
    return loss

# L1Loss Mean Absolute Error
def L1Loss(predict, target):
    loss = torch.nn.L1Loss()(predict, target)
    return loss

# BCEWithLogitsLoss
def BCEWithLogitsLoss(predict, target):
    loss = torch.nn.BCEWithLogitsLoss()(predict, target)
    return loss

# NLLLoss(多分类)
def NLLLoss(predict, target):
    loss = torch.nn.NLLLoss()(predict, target)
    return loss

# SmoothL1Loss(多分类)
def SmoothL1Loss(predict, target):
    loss = torch.nn.SmoothL1Loss()(predict, target)
    return loss

# DiceLoss
class DiceLoss(nn.Module):
    def __init__(self, weight=None, size_average=True):
        super(DiceLoss, self).__init__()

    def forward(self, inputs, targets, smooth=1):
        # comment out if your model contains a sigmoid or equivalent activation layer
        inputs = F.sigmoid(inputs)

        # flatten label and prediction tensors
        inputs = inputs.view(-1)
        targets = targets.view(-1)

        intersection = (inputs * targets).sum()
        dice = (2. * intersection + smooth) / (inputs.sum() + targets.sum() + smooth)

        return 1 - dice

# DiceMeanLoss
class DiceMeanLoss(nn.Module):
    def __init__(self):
        super(DiceMeanLoss, self).__init__()

    def forward(self, logits, targets):
        class_num = logits.size()[1]

        dice_sum = 0
        for i in range(class_num):
            inter = torch.sum(logits[:, i, :, :, :] * targets[:, i, :, :, :])
            union = torch.sum(logits[:, i, :, :, :]) + torch.sum(targets[:, i, :, :, :])
            dice = (2. * inter + 1) / (union + 1)
            dice_sum += dice
        return 1 - dice_sum / class_num

# FocalLoss
class FocalLoss(nn.Module):
    def __init__(self, weight=None, size_average=True):
        super(FocalLoss, self).__init__()
    ALPHA = 0.8
    GAMMA = 2

    def forward(self, inputs, targets, alpha=ALPHA, gamma=GAMMA, smooth=1):
        # comment out if your model contains a sigmoid or equivalent activation layer
        inputs = F.sigmoid(inputs)

        # flatten label and prediction tensors
        inputs = inputs.view(-1)
        targets = targets.view(-1)

        # first compute binary cross-entropy
        BCE = F.binary_cross_entropy(inputs, targets, reduction='mean')
        BCE_EXP = torch.exp(-BCE)
        focal_loss = alpha * (1 - BCE_EXP) ** gamma * BCE

        return focal_loss

# --------------------------- BINARY LOSSES ---------------------------
class BFocalLoss(nn.Module):
    def __init__(self, alpha=0.8, gamma=2, weight=None, ignore_index=255):
        super(BFocalLoss, self).__init__()
        self.alpha = alpha
        self.gamma = gamma
        self.weight = weight
        self.ignore_index = ignore_index
        self.bce_fn = nn.BCEWithLogitsLoss(weight=self.weight)

    def forward(self, preds, labels):
        if self.ignore_index is not None:
            mask = labels != self.ignore_index
            labels = labels[mask]
            preds = preds[mask]

        logpt = -self.bce_fn(preds, labels)
        pt = torch.exp(logpt)
        loss = -((1 - pt) ** self.gamma) * self.alpha * logpt
        return loss
# --------------------------- MULTICLASS LOSSES ---------------------------
class MFocalLoss(nn.Module):
    def __init__(self, alpha=0.8, gamma=2, weight=None, ignore_index=255):
        super(MFocalLoss, self).__init__()
        self.alpha = alpha
        self.gamma = gamma
        self.weight = weight
        self.ignore_index = ignore_index
        self.ce_fn = nn.CrossEntropyLoss(weight=self.weight, ignore_index=self.ignore_index)

    def forward(self, preds, labels):
        logpt = -self.ce_fn(preds, labels)
        pt = torch.exp(logpt)
        loss = -((1 - pt) ** self.gamma) * self.alpha * logpt
        return loss

# IoULoss
class IoULoss(nn.Module):
    def __init__(self, weight=None, size_average=True):
        super(IoULoss, self).__init__()

    def forward(self, inputs, targets, smooth=1):
        # comment out if your model contains a sigmoid or equivalent activation layer
        inputs = F.sigmoid(inputs)

        # flatten label and prediction tensors
        inputs = inputs.view(-1)
        targets = targets.view(-1)

        # intersection is equivalent to True Positive count
        # union is the mutually inclusive area of all labels & predictions
        intersection = (inputs * targets).sum()
        total = (inputs + targets).sum()
        union = total - intersection

        IoU = (intersection + smooth) / (union + smooth)

        return 1 - IoU

def main():
    import numpy

    print(log_loss(0, 0))
    print(log_loss(0, 1))

    predict = torch.rand(2, 2, 24, 24, 48)  # predict是预测结果
    target = torch.empty(2, 2, 24, 24, 48).random_(0, 2)  # target是ground true

    print(BCEWithLogitsLoss(predict, target))

    predict = torch.sigmoid(predict)

    print(DiceLoss()(predict, target))
    print(DiceMeanLoss()(predict, target))
    print(FocalLoss()(predict, target))
    print(BFocalLoss()(predict, target))
    print(IoULoss()(predict, target))

    print(BCELoss(predict, target))
    print(MSELoss(predict, target))

    target = torch.empty(2, 24, 24, 48).random_(0, 2).long()  # target是ground true
    print(NLLLoss(predict, target))
    print(CrossEntropyLoss(predict, target))
    print(MFocalLoss()(predict, target))

if __name__ == '__main__':
    main()


import numpy as np
import matplotlib.pyplot as plt
import math as mt

def cond_check(m):
    p=np.array([4,7])
    return np.matmul(p,m)**2 >=(np.linalg.norm(p)**2-9)

num=0
radius=3
p=np.array([4,7])
norm_p=np.linalg.norm(p)
m_lim1=p/mt.sqrt(norm_p**2-9)
m_lim2=-p/mt.sqrt(norm_p**2-9)
theta_line=np.linspace(0,mt.pi/2,20)

for theta in theta_line:
    m=np.array([mt.cos(theta),mt.sin(theta)])
    if cond_check(m):
        lambda_1=-np.matmul(p,m)+mt.sqrt((np.matmul(p,m)**2)+9-norm_p**2)
        lambda_2=-np.matmul(p,m)-mt.sqrt((np.matmul(p,m)**2)+9-norm_p**2)
        A=p+lambda_1*m
        B=p+lambda_2*m
        PA_PB=np.linalg.norm(p-A)*np.linalg.norm(p-B)
        print('m=',m,'   '+'PA.PB=',PA_PB)
        plt.plot(np.array([p[0],A[0],B[0]]),np.array([p[1],A[1],B[1]]))
        plt.text(A[0]*(1+0.1),A[1]*(1-0.1),'A'+str(num))
        plt.text(B[0]*(1+0.1),B[1]*(1-0.1),'B'+str(num))
        num=num+1
theta=np.linspace(0,2*mt.pi,100)
plt.plot(radius*np.cos(theta),radius*np.sin(theta))
plt.plot(p[0],p[1])
plt.text(p[0],p[1],'P')
plt.grid()
plt.show()


import lxml.html
import requests
from bs4 import BeautifulSoup as bs
from datetime import datetime
from subprocess import Popen, STARTUPINFO

'''Path for the file to be written'''

FILE_PATH = 'E:\\Documents\\promocoes.txt'
BACKUP_FILE_PATH = 'E:\Documents\\promocoes_backup.txt'

'''Start up options for process
wShowWindow = 0 -> hidden'''

start_info = STARTUPINFO()
start_info.dwFlags = 1
start_info.wShowWindow = 1

'''current time'''
time = datetime.now()

def get_html_cloudfare(link):
    scraper = cfscrape.create_scraper()
    return scraper.get(link).content

def get_title(soup):
    return soup.title.encode('utf-8')

def complete_links(partial_url, base_url):
    return '{}/{}'.format(base_url, partial_url)

def get_dom(scrap_url):
    '''html_link -> lxml dom'''
    html_doc = requests.get(scrap_url)
    return lxml.html.fromstring(html_doc.content)

def get_soup(link):
    '''html link -> BeautifulSoup soup'''
    html_doc = requests.get(link)
    return bs(html_doc.content)

def format_adrenaline_link(link):
    '''link (adrenaline) -> formatted link
    '''
    _, t = link.rsplit('/', 1)
    thread_title, _ = t.split('.')
    formatted_title = ' '.join([word for word in thread_title.split('-')])
    return "\nPromo: {}\nLink: {}\n".format(formatted_title.replace(' r ', ' R$').title(), link)

def format_hardmob_links(link):
    '''
    link: hardmob link
    link -> formatted link'''
    _, title = link.split('-', 1)
    c, _ = title.split('.', 1)
    formatted_title = ' '.join(
        [word for word in c.split('-')[1:] if word is not '-'])
    return "\nPromo: {}\nLink: {}\n".format(formatted_title.replace(' r ', ' R$').title(), link)

def gen_adrenaline_links(scrap_url='http://adrenaline.uol.com.br/forum/forums/for-sale.221/', base_url='http://adrenaline.uol.com.br/forum', key_word='thread', n_links=10):
    '''scrap_url: adrenaline url to be scrapped
    n_links: max number of links returned
    base_url: base adrenaline url
    key_word: word to filter pertinent forum entries
    url -> formatted links'''
    dom = get_soup(scrap_url)
    links_list = set()
    for link in dom.findAll('a'):
        href = link.get('href')
        if href:
            if key_word in href and not any(word in href for word in['atencao', 'regras', 'censurados']):
                link, _ = href.rsplit('/', 1)
                links_list.add(complete_links(link, base_url))
            elif len(links_list) == n_links:
                break
    for link in set(links_list):
        corrected_link = format_adrenaline_link(link)
        if corrected_link:
            yield corrected_link

def get_hardmob_links(scrap_url='http://www.hardmob.com.br/promocoes', n_links=5):
    '''scrap_url: hardmob url to be scrapped
    n_links: max number of links returned
    url -> formatted links'''
    dom = get_soup(scrap_url)
    link_count = 0
    for link in dom.findAll('a'):
        if link.get('id'):
            if 'thread_title' in link.get('id') and 'twitter' not in link.get('href') and 'faq' not in link.get('href') and 'descontos' not in link.get('href'):
                thread_link = link.get('href')
                try:
                    FILE.write(format_hardmob_links(thread_link))
                except Exception as e:
                    FILE.write(e)
                link_count += 1
            if link_count == n_links:
                break

def gen_promoforum_links(scrap_url='http://www.promoforum.com.br/forums/promocoes', n_links=5):
    '''scrap_url: promoforum url to be scrapped
    n_links: max number of links returned
    url -> formatted links'''
    base_link = 'http://www.promoforum.com.br/'
    dom = get_soup(scrap_url)
    link_count = 0
    for link in dom.findAll('a'):
        if link.get('class'):
            if link.get('class')[0] == 'PreviewTooltip' and 'postar' not in link.get('href'):
                thread_link = link.get('href')
                yield format_hardmob_links(complete_links(thread_link.encode('utf-8'), base_link))
                link_count += 1
            if link_count == n_links:
                break

def gen_promobit_links(scrap_url='http://www.promobit.com.br/Promocoes/em-destaque/page/1-2', n_links=5):
    '''scrap_url: promobit url to be scrapped
    n_links: max number of links returned
    url -> formatted links'''
    soup = get_soup(scrap_url)
    base_link = 'http://www.promobit.com.br'
    for link in soup.findAll('a'):
        try:
            href = link.get('href').encode('utf-8')
            info = link.text.encode('utf-8')
            if 'ver-promocoes' in href and 'R$' in info:
                title, price = link.text.encode('utf-8').split('R$', 1)
                formatted_title = "\nPromo: {} por R${}Link: {}{}\n".format(
                    title.strip(), price.rsplit(' ', 1)[1], base_link, href)
                yield formatted_title
        except AttributeError:
            continue

def populate_txt(links, cmp_file=BACKUP_FILE_PATH):
    '''checks if link is already on file
    links: formatted promotion links to be written in the file
    links (generator/list) -> write txt'''
    with open(cmp_file, 'a+') as f:
        txt = f.read()
    for link in links:
        if link not in txt:
            with open(FILE_PATH, 'a+') as f:
                f.write(link)

def main():
    with open(FILE_PATH, 'r+') as f:
        with open(BACKUP_FILE_PATH, 'a+') as f2:
            old_text = f.read()
            f2.write(old_text)
            f.seek(0)
            f.truncate()
    populate_txt(gen_promobit_links(n_links=2))
    populate_txt(gen_promoforum_links(n_links=2))
    populate_txt(gen_adrenaline_links(n_links=2))
    with open(FILE_PATH, 'r') as g:
        new_text = g.read()
    if new_text:
        Popen(["notepad.exe", FILE_PATH])

if __name__ == '__main__':
    main()


from ComputerVision.Values_Transformations.filter_kernel import sobel_operators, box_kernel
from ComputerVision.Values_Transformations.filter import Filter
from tqdm.notebook import tqdm
import numpy as np

class CannyEdge:
    """
    Canny Edge Detection algorithm
    """

    @staticmethod
    def group_angle(x):
        """
        Group angles by groups
        :param x: angle
        :return: new angle
        """
        if -22.5 <= x <= 22.5 or 157.5 <= x or -157.5 >= x:
            return 0
        if 22.5 <= x <= 67.5 or -112.5 >= x >= -157.5:
            return 45
        if 67.5 <= x <= 112.5 or -67.5 >= x >= -112.5:
            return 90
        if 112.5 <= x <= 157.5 or -22.5 >= x >= -67.5:
            return 135

    @staticmethod
    def compare_magnitude(x, y, i, j):
        """
        Magnitude comparison
        :param x: magnitude
        :param y: image
        :param i: coordinate image x
        :param j: coordinate image y
        :return: value
        """
        if x == 0:
            if y[i, j] < y[i - 1, j] or y[i, j] < y[i + 1, j]:
                return 0
        if x == 45:
            if y[i, j] < y[i - 1, j + 1] or y[i, j] < y[i + 1, j - 1]:
                return 0
        if x == 90:
            if y[i, j] < y[i - 1, j] or y[i, j] < y[i + 1, j]:
                return 0
        if x == 135:
            if y[i, j] < y[i - 1, j] or y[i, j] < y[i + 1, j]:
                return 0
        return y[i, j]

    @staticmethod
    def compare_magnitudes(x, y):
        """
        Compare magnitudes
        :param x: image x
        :param y: image y
        :return: new image
        """
        aux = np.zeros(x.shape)
        for i in tqdm(range(x.shape[0])):
            for j in range(x.shape[1]):
                try:
                    aux[i, j] = CannyEdge.compare_magnitude(x[i, j], y, i, j)
                except IndexError:
                    pass
        return aux

    @staticmethod
    def thresholding(x, th, tl):
        """
        Apply thresholds
        :param x: image
        :param th: small threshold
        :param tl: big threshold
        :return: value
        """
        if x >= th:
            return 2
        if th > x >= tl:
            return 1
        return 0

    @staticmethod
    def get_neighbours(i, j, shape_):
        """
        Get neighbours pixel
        :param i: coordinate x
        :param j: coordinate y
        :param shape_: shape image
        :return: neighbours
        """
        list_indexes = [(i, j)]
        if i - 1 >= 0:
            if j - 1 >= 0:
                list_indexes.append((i - 1, j - 1))
            if j >= 0:
                list_indexes.append((i - 1, j))
            if j + 1 < shape_[1]:
                list_indexes.append((i - 1, j + 1))
        if j - 1 >= 0:
            list_indexes.append((i, j - 1))
        if j + 1 < shape_[1]:
            list_indexes.append((i, j + 1))
        if i + 1 < shape_[0]:
            if j - 1 >= 0:
                list_indexes.append((i + 1, j - 1))
            if j >= 0:
                list_indexes.append((i + 1, j))
            if j + 1 < shape_[1]:
                list_indexes.append((i + 1, j + 1))
        return list_indexes

    @staticmethod
    def region_grow(f, f5):
        """
        Do region grow
        :param f: filter function
        :param f5: image
        :return: new image
        """
        while True:
            xy = np.nonzero(f(f5))
            if len(xy[0]) > 0:
                x = xy[0][0]
                y = xy[1][0]
                num = 0
                for (i, j) in CannyEdge.get_neighbours(x, y, f5.shape):
                    if f5[i, j] == 2:
                        num += 1
                        f5[x, y] = 2
                        break
                if num == 0:
                    f5[x, y] = 0
            else:
                break
        return f5

    @staticmethod
    def apply(img, tl=0.05, th=0.15, do_grow=True, kernel_size=3):
        """
        Apply algorithm
        :param img: image
        :param tl: low threshold
        :param th: upper threshold
        :param do_grow: apply incremental grow
        :param kernel_size: kernel size
        :return: new image
        """
        print("Applying Canny edge detector")
        # Step 1
        print("    (1/5) Integration")
        g = box_kernel(kernel_size)
        f1 = Filter.apply_filter(img, g)

        # Step 2
        print("    (2/5) Differentiation")
        # Sobbel
        g_x, g_y = sobel_operators()
        f2_x = Filter.apply_filter(f1, g_x)
        f2_y = Filter.apply_filter(f1, g_y)

        # Magnitude
        G = np.sqrt(f2_x ** 2 + f2_y ** 2)
        # Orientation
        # to don't divide by 0
        f = np.vectorize(lambda x: 10e-19 if x == 0. else x)
        teta = np.degrees(np.arctan(f(f2_y) / f(f2_x)))

        # Step 3
        print("    (3/5) Non-maxima suppression")
        # Group angles
        f = np.vectorize(lambda x: CannyEdge.group_angle(x))
        teta2 = f(teta)
        f3 = CannyEdge.compare_magnitudes(teta2, G)

        # Step 4
        print("    (4/5) Hysteresis (or Dual) thresholding")
        f = np.vectorize(CannyEdge.thresholding)
        f3n = f3 / np.max(f3)  # Normalize
        f4 = f(f3n, th, tl)

        # Step 5
        print("    (5/5) Connectivity analysis")
        f = np.vectorize(lambda x: 0 if x == 2 else x)
        f5 = np.copy(f4)
        if do_grow:
            f5 = CannyEdge.region_grow(f, f5)
        print("Complete!")
        return f5


"""
    Description : This Program detect diabetes using Machine Learning and Python
"""
# run block of code and catch warnings
import warnings

with warnings.catch_warnings():
    warnings.filterwarnings("ignore")

import pandas as pd
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
import streamlit as st

st.write("""
# Diabetes Detection
Application of Python in Real-World Situations
""")

data = pd.read_csv("diabetes.csv")

st.subheader("Data Information")

st.dataframe(data)

st.write(data.describe())

chart = st.bar_chart(data)

X = data.iloc[: , 0:8].values
Y = data.iloc[:, -1].values

x_train,x_test,y_train,y_test = train_test_split(X,Y,test_size = 0.25, random_state = 0)

def get_user_input():
    pregnancies = st.sidebar.slider("Pregnancies",0,17,3)
    glucose = st.sidebar.slider("Glucose",0,199,117)
    blood_pressure = st.sidebar.slider("Blood Pressure",0,122,72)
    skin_thickness = st.sidebar.slider("Skin Thickness",0,99,23)
    insulin = st.sidebar.slider("Insulin",0.0,846.0,30.0)
    bmi = st.sidebar.slider("BMI",0.0,67.1,32.0)
    diabetes_pedigree_function = st.sidebar.slider("Diabetes Pedigree Function",0.078,2.42,0.3725)
    age = st.sidebar.slider("Age",0,200,29)

    user_data = {"pregnancies": pregnancies,
                 "glucose" : glucose,
                 "blood_pressure" : blood_pressure,
                 "skin_thickness" : skin_thickness,
                 "insulin" : insulin,
                 "bmi" : bmi,
                 "diabetes_pedigree_function" : diabetes_pedigree_function,
                 "age" : age
                 }
    features = pd.DataFrame(user_data, index = [0])
    return features

user_input = get_user_input()

st.subheader("User Input :")
st.write(user_input)

model = RandomForestClassifier(n_estimators = 100)
model.fit(x_train,y_train)

#st.subheader("Model Test Accuracy Score:")
#st.write(str(accuracy_score(y_test) , model.predict(x_test))*100 + "%")

predictions = model.predict(user_input)

if str(predictions) == '[0]':
    st.subheader("Predictions")
    st.write("The person is not diabetic.")
elif str(predictions) == '[1]':
    st.subheader("Predictions")
    st.write("The person is diabetic.")


import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torchvision.utils as utils
from torchvision import datasets, transforms
from torch.utils.data import DataLoader

device = 'cuda' if torch.cuda.is_available() else 'cpu'
batch_size = 500

# The output of torchvision datasets are PILImage images of range [0, 1].
# We transform them to Tensors of normalized range [-1, 1].
transform = transforms.Compose(
    [transforms.ToTensor(),
     transforms.Normalize((0.5), (0.5))])

# MNIST Dataset
train_dataset = datasets.MNIST(root='data/', train=True, transform=transform, download=True)
test_dataset = datasets.MNIST(root='data/', train=False, transform=transform)

# Data Loader
train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)
test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False)

class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.l1 = nn.Linear(784, 512)
        self.l2 = nn.Linear(512, 240)
        self.l3 = nn.Linear(240, 10)

    def forward(self, input):
        output = input.view(-1, 784) # Flatten the data (n, 1, 28, 28)-> (n, 784)
        output = F.relu(self.l1(output))
        output = F.relu(self.l2(output))
        output = self.l3(output)
        return output

model = Model()
model.to(device)

criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.003)

def train(epoch):
    model.train()

    for batch_idx, (data, label) in enumerate(train_loader):
        data, label = data.to(device), label.to(device)

        optimizer.zero_grad()
        output = model(data)
        loss = criterion(output, label)
        loss.backward()
        optimizer.step()

        if batch_idx % 10 == 0:
            print('Train Epoch: {} | Batch Status: {}/{} ({:.0f}%) | Loss: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader), loss.item()))

def test():
    model.eval()
    test_loss = 0
    correct = 0

    for data, label in test_loader:
        data, label = data.to(device), label.to(device)

        output = model(data)
        test_loss += criterion(output, label).item()
        pred = output.data.max(1, keepdim=True)[1]
        correct += pred.eq(label.data.view_as(pred)).cpu().sum()

    test_loss /= len(test_loader.dataset)
    print(f'===========================\nTest set: Average loss: {test_loss:.4f}, Accuracy: {correct}/{len(test_loader.dataset)} '
          f'({100. * correct / len(test_loader.dataset):.0f}%)')

for epoch in range(3):
    train(epoch)
    test()


from typing import Literal, TypedDict
from urllib.parse import urljoin

from scrapy import Request, Spider
from scrapy.selector import Selector

from csv_writer import CSVWriter

PartOfSpeech = Literal["Verb", "Adjektiv", "Substantiv"]

class LemmaFilters(TypedDict, total=False):
    part_of_speech: set[PartOfSpeech]
    starts_with: str
    does_not_start_with: list[str]

LemmaFeature = Literal["url", "title", "part_of_speech", "hyphenation"]

SOFT_HYPHEN = "\xad"

class DudenSpider(Spider):
    """Extract Information from duden.de

    Spider has 2 levels: 1st one handles search results, 2nd one lemma page.
    """

    name = "duden spide"
    base_url = "https://www.duden.de/"
    filters: LemmaFilters
    features: list[LemmaFeature]

    def parse(
        self, search_result_page
    ):  # pylint: disable=arguments-renamed, arguments-differ
        lemma_outline_path = "*//section[@class='vignette']"
        outlines = search_result_page.xpath(lemma_outline_path)
        for lemma_outline in outlines:
            if self._lemma_outline_is_relevant(lemma_outline):
                url = urljoin(
                    self.base_url,
                    self._extract_relative_lemma_url(lemma_outline),
                )
                yield Request(
                    url=url,
                    callback=self.parse_lemma_page,
                )
        next_relative_url = self._extract_next_page_url(search_result_page)
        if next_relative_url:
            yield Request(
                url=urljoin(
                    search_result_page.request.url,
                    next_relative_url,
                ),
                callback=self.parse,
            )

    @staticmethod
    def _extract_next_page_url(search_result_page) -> str:
        next_page_link_path = "//a[@class='pager__item' and @rel='next']/@href"
        result = search_result_page.xpath(next_page_link_path).extract()
        if not result:
            return None

        return result[0]

    @staticmethod
    def _clean_soft_hyphens(word: str) -> str:
        return word.replace(SOFT_HYPHEN, "")

    def _lemma_outline_is_relevant(self, lemma_outline: Selector) -> bool:
        """Outline is relevant if it does not contradict filters"""

        lemma_title = self._extract_lemma_title(lemma_outline)
        if "starts_with" in self.filters:
            if not any(
                lemma_title.startswith(mandatory_start)
                for mandatory_start in self.filters["starts_with"]
            ):
                return False

        if "does_not_start_with" in self.filters:
            if any(
                lemma_title.startswith(forbidden_beginning)
                for forbidden_beginning in self.filters["does_not_start_with"]
            ):
                return False

        if "part_of_speech" in self.filters:
            description = self._extract_lemma_description(lemma_outline)
            if not any(
                description.startswith(relevant_part_of_speech)
                for relevant_part_of_speech in self.filters["part_of_speech"]
            ):
                return False

        return True

    def _extract_lemma_title(self, lemma_outline: Selector) -> str:
        title_path = "h2[@class='vignette__title']/a/strong/text()"
        result = lemma_outline.xpath(title_path).extract()[0]
        return self._clean_soft_hyphens(result)

    @staticmethod
    def _extract_lemma_description(lemma_outline: Selector) -> str:
        description_path = "p[@class='vignette__snippet']/text()"
        return lemma_outline.xpath(description_path).extract()[0].strip()

    @staticmethod
    def _extract_relative_lemma_url(lemma_outline: Selector) -> str:
        url_path = "a[@class='vignette__link']/@href"
        relative_url = lemma_outline.xpath(url_path).extract()[0]
        return relative_url

    def parse_lemma_page(self, lemma_page) -> None:
        result = {}
        if "url" in self.features:
            result["url"] = lemma_page.request.url
        if "title" in self.features:
            result["title"] = self._extract_title(lemma_page)
        if "hyphenation" in self.features:
            result["hyphenation"] = self._extract_hyphenation(lemma_page)
        if "part_of_speech" in self.features:
            result["part_of_speech"] = self._extract_pos(lemma_page)
        self.csv_writer.writerow(result)

    @staticmethod
    def _extract_pos(lemma_page) -> str | None:
        pos_path = (
            "//article/dl["
            "dt["
            "@class='tuple__key' "
            "and contains(text(), 'Wortart')"
            "]"
            "]/dd[@class='tuple__val']/text()"
        )
        pos = lemma_page.xpath(pos_path).extract()
        if not pos:
            return None
        return pos[0]

    @staticmethod
    def _extract_raw_title(lemma_page) -> str:
        title_path = "//div[@class='lemma']/h1/span/text()"
        return lemma_page.xpath(title_path).extract()[0]

    def _extract_title(self, lemma_page) -> str:
        raw_title = self._extract_raw_title(lemma_page)
        return self._clean_soft_hyphens(raw_title)

    def _extract_hyphenation(self, lemma_page) -> str:
        raw_title = self._extract_raw_title(lemma_page)
        hyphenation = raw_title.replace(SOFT_HYPHEN, " | ")
        return hyphenation

def make_duden_spider(
    output_path: str,
    search_term: str,
    features: list[LemmaFeature],
    filters: LemmaFilters | None = None,
) -> DudenSpider:
    filters = filters or {}
    ConfiguredDudenSpider = DudenSpider
    ConfiguredDudenSpider.filters = filters
    ConfiguredDudenSpider.features = features

    ConfiguredDudenSpider.start_urls = [
        f"https://www.duden.de/suchen/dudenonline/{search_term}"
    ]
    ConfiguredDudenSpider.csv_writer = CSVWriter(output_path, fields=features)
    return ConfiguredDudenSpider


import os
import cv2
import face_recognition
import numpy as np
from tqdm import tqdm
from collections import defaultdict
from imutils.video import VideoStream
from eye_status import *

def init():
    face_cascPath = 'haarcascade_frontalface_alt.xml'
    # face_cascPath = 'lbpcascade_frontalface.xml'

    open_eye_cascPath = 'haarcascade_eye_tree_eyeglasses.xml'
    left_eye_cascPath = 'haarcascade_lefteye_2splits.xml'
    right_eye_cascPath ='haarcascade_righteye_2splits.xml'
    dataset = 'faces'

    face_detector = cv2.CascadeClassifier(face_cascPath)
    open_eyes_detector = cv2.CascadeClassifier(open_eye_cascPath)
    left_eye_detector = cv2.CascadeClassifier(left_eye_cascPath)
    right_eye_detector = cv2.CascadeClassifier(right_eye_cascPath)

    print("[LOG] Opening webcam ...")
    video_capture = VideoStream(src=0).start()

    model = load_model()

    print("[LOG] Collecting images ...")
    images = []
    for direc, _, files in tqdm(os.walk(dataset)):
        for file in files:
            if file.endswith("jpg"):
                images.append(os.path.join(direc,file))
    return (model,face_detector, open_eyes_detector, left_eye_detector,right_eye_detector, video_capture, images)

def process_and_encode(images):
    # initialize the list of known encodings and known names
    known_encodings = []
    known_names = []
    print("[LOG] Encoding faces ...")

    for image_path in tqdm(images):
        # Load image
        image = cv2.imread(image_path)
        # Convert it from BGR to RGB
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

        # detect face in the image and get its location (square boxes coordinates)
        boxes = face_recognition.face_locations(image, model='hog')

        # Encode the face into a 128-d embeddings vector
        encoding = face_recognition.face_encodings(image, boxes)

        # the person's name is the name of the folder where the image comes from
        name = image_path.split(os.path.sep)[-2]

        if len(encoding) > 0 :
            known_encodings.append(encoding[0])
            known_names.append(name)

    return {"encodings": known_encodings, "names": known_names}

def isBlinking(history, maxFrames):
    """ @history: A string containing the history of eyes status
         where a '1' means that the eyes were closed and '0' open.
        @maxFrames: The maximal number of successive frames where an eye is closed """
    for i in range(maxFrames):
        pattern = '1' + '0'*(i+1) + '1'
        if pattern in history:
            return True
    return False

def detect_and_display(model, video_capture, face_detector, open_eyes_detector, left_eye_detector, right_eye_detector, data, eyes_detected):
        frame = video_capture.read()
        # resize the frame
        frame = cv2.resize(frame, (0, 0), fx=0.6, fy=0.6)

        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

        # Detect faces
        faces = face_detector.detectMultiScale(
            gray,
            scaleFactor=1.2,
            minNeighbors=5,
            minSize=(50, 50),
            flags=cv2.CASCADE_SCALE_IMAGE
        )

        # for each detected face
        for (x,y,w,h) in faces:
            # Encode the face into a 128-d embeddings vector
            encoding = face_recognition.face_encodings(rgb, [(y, x+w, y+h, x)])[0]

            # Compare the vector with all known faces encodings
            matches = face_recognition.compare_faces(data["encodings"], encoding)

            # For now we don't know the person name
            name = "Unknown"

            # If there is at least one match:
            if True in matches:
                matchedIdxs = [i for (i, b) in enumerate(matches) if b]
                counts = {}
                for i in matchedIdxs:
                    name = data["names"][i]
                    counts[name] = counts.get(name, 0) + 1

                # determine the recognized face with the largest number of votes
                name = max(counts, key=counts.get)

            face = frame[y:y+h,x:x+w]
            gray_face = gray[y:y+h,x:x+w]

            eyes = []

            # Eyes detection
            # check first if eyes are open (with glasses taking into account)
            open_eyes_glasses = open_eyes_detector.detectMultiScale(
                gray_face,
                scaleFactor=1.1,
                minNeighbors=5,
                minSize=(30, 30),
                flags = cv2.CASCADE_SCALE_IMAGE
            )
            # if open_eyes_glasses detect eyes then they are open
            if len(open_eyes_glasses) == 2:
                eyes_detected[name]+='1'
                for (ex,ey,ew,eh) in open_eyes_glasses:
                    cv2.rectangle(face,(ex,ey),(ex+ew,ey+eh),(0,255,0),2)

            # otherwise try detecting eyes using left and right_eye_detector
            # which can detect open and closed eyes
            else:
                # separate the face into left and right sides
                left_face = frame[y:y+h, x+int(w/2):x+w]
                left_face_gray = gray[y:y+h, x+int(w/2):x+w]

                right_face = frame[y:y+h, x:x+int(w/2)]
                right_face_gray = gray[y:y+h, x:x+int(w/2)]

                # Detect the left eye
                left_eye = left_eye_detector.detectMultiScale(
                    left_face_gray,
                    scaleFactor=1.1,
                    minNeighbors=5,
                    minSize=(30, 30),
                    flags = cv2.CASCADE_SCALE_IMAGE
                )

                # Detect the right eye
                right_eye = right_eye_detector.detectMultiScale(
                    right_face_gray,
                    scaleFactor=1.1,
                    minNeighbors=5,
                    minSize=(30, 30),
                    flags = cv2.CASCADE_SCALE_IMAGE
                )

                eye_status = '1' # we suppose the eyes are open

                # For each eye check wether the eye is closed.
                # If one is closed we conclude the eyes are closed
                for (ex,ey,ew,eh) in right_eye:
                    color = (0,255,0)
                    pred = predict(right_face[ey:ey+eh,ex:ex+ew],model)
                    if pred == 'closed':
                        eye_status='0'
                        color = (0,0,255)
                    cv2.rectangle(right_face,(ex,ey),(ex+ew,ey+eh),color,2)
                for (ex,ey,ew,eh) in left_eye:
                    color = (0,255,0)
                    pred = predict(left_face[ey:ey+eh,ex:ex+ew],model)
                    if pred == 'closed':
                        eye_status='0'
                        color = (0,0,255)
                    cv2.rectangle(left_face,(ex,ey),(ex+ew,ey+eh),color,2)
                eyes_detected[name] += eye_status

            # Each time, we check if the person has blinked
            # If yes, we display its name
            if isBlinking(eyes_detected[name],3):
                cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 255, 0), 2)
                # Display name
                y = y - 15 if y - 15 > 15 else y + 15
                cv2.putText(frame, name, (x, y), cv2.FONT_HERSHEY_SIMPLEX,0.75, (0, 255, 0), 2)

        return frame

if __name__ == "__main__":
    (model, face_detector, open_eyes_detector,left_eye_detector,right_eye_detector, video_capture, images) = init()
    data = process_and_encode(images)

    eyes_detected = defaultdict(str)
    while True:
        frame = detect_and_display(model, video_capture, face_detector, open_eyes_detector,left_eye_detector,right_eye_detector, data, eyes_detected)
        cv2.imshow("Face Liveness Detector", frame)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break
    cv2.destroyAllWindows()
    video_capture.stop()