import os
from typing import Tuple, Union

import pathlib

import torch
import torchvision
import yacs.config

from torch.utils.data import Dataset

from pytorch_image_classification import create_transform

# DATASETS_ROOT = "/root/autodl-tmp"
DATASETS_ROOT = "/media/joshuawen/Joshua_SSD3/Datasets/RGB/classification"
CUSTOM_DATASETS = ['RS19-WHU', 'RSSCN7', 'Google-China', 'EuroSAT', 'AID',
                   'UCMerced_LandUse', 'RSD46-WHU', 'PatternNet', 'RSI-CB128', 'RSI-CB256']

class SubsetDataset(Dataset):
    def __init__(self, subset_dataset, transform=None):
        self.subset_dataset = subset_dataset
        self.transform = transform

    def __getitem__(self, index):
        x, y = self.subset_dataset[index]
        if self.transform:
            x = self.transform(x)
        return x, y

    def __len__(self):
        return len(self.subset_dataset)

def create_dataset(config: yacs.config.CfgNode,
                   is_train: bool) -> Union[Tuple[Dataset, Dataset], Dataset]:
    if config.dataset.name in [
            'CIFAR10',
            'CIFAR100',
            'MNIST',
            'FashionMNIST',
            'KMNIST',
    ]:
        module = getattr(torchvision.datasets, config.dataset.name)
        if is_train:
            if config.train.use_test_as_val:
                train_transform = create_transform(config, is_train=True)
                val_transform = create_transform(config, is_train=False)
                train_dataset = module(config.dataset.dataset_dir,
                                       train=is_train,
                                       transform=train_transform,
                                       download=True)
                test_dataset = module(config.dataset.dataset_dir,
                                      train=False,
                                      transform=val_transform,
                                      download=True)
                return train_dataset, test_dataset
            else:
                dataset = module(config.dataset.dataset_dir,
                                 train=is_train,
                                 transform=None,
                                 download=True)
                val_ratio = config.train.val_ratio
                assert val_ratio < 1
                val_num = int(len(dataset) * val_ratio)
                train_num = len(dataset) - val_num
                lengths = [train_num, val_num]
                train_subset, val_subset = torch.utils.data.dataset.random_split(
                    dataset, lengths)

                train_transform = create_transform(config, is_train=True)
                val_transform = create_transform(config, is_train=False)
                train_dataset = SubsetDataset(train_subset, train_transform)
                val_dataset = SubsetDataset(val_subset, val_transform)
                return train_dataset, val_dataset
        else:
            transform = create_transform(config, is_train=False)
            dataset = module(config.dataset.dataset_dir,
                             train=is_train,
                             transform=transform,
                             download=True)
            return dataset
    elif config.dataset.name == 'ImageNet':
        dataset_dir = pathlib.Path(config.dataset.dataset_dir).expanduser()
        train_transform = create_transform(config, is_train=True)
        val_transform = create_transform(config, is_train=False)
        train_dataset = torchvision.datasets.ImageFolder(
            dataset_dir / 'train', transform=train_transform)
        val_dataset = torchvision.datasets.ImageFolder(dataset_dir / 'val',
                                                       transform=val_transform)
        return train_dataset, val_dataset
    elif config.dataset.name in CUSTOM_DATASETS:
        train_transform = create_transform(config, is_train=True)
        val_transform = create_transform(config, is_train=False)
        train_dataset = torchvision.datasets.ImageFolder(
            os.path.join(DATASETS_ROOT, config.dataset.name, 'train'), transform=train_transform)
        val_dataset = torchvision.datasets.ImageFolder(
            os.path.join(DATASETS_ROOT, config.dataset.name, 'val'), transform=val_transform)
        return train_dataset, val_dataset
    else:
        raise ValueError()


import numpy as np
import os
import gym
import torch
import torch.nn as nn
from torch.autograd import Variable

USE_CUDA = torch.cuda.is_available()

def get_checkpoint_file(d, fid):
    return os.path.join(d, 'checkpoint{:d}'.format(fid))

def make_game():
    return gym.make('Pong-v0')

def to_tensor_f32(x):
    t_ = torch.from_numpy(np.float32(np.ascontiguousarray(x)))
    if USE_CUDA:
        t_ = t_.cuda()
    return t_

def to_tensor_int(x):
    # noinspection PyArgumentList
    t_ = torch.LongTensor(x)
    if USE_CUDA:
        t_ = t_.cuda()
    return t_

def to_numpy(t):
    if USE_CUDA:
        x = t.cpu().numpy()
    else:
        x = t.numpy()
    return x

# hyperparameters
H = 200  # number of hidden layer neurons
D = 80 * 80  # input dimensionality: 80x80 grid

# noinspection PyPep8Naming,PyPep8Naming
def prepro(I):
    """ prepro 210x160x3 uint8 frame into 6400 (80x80) 1D float vector """
    I = I[35:195]  # crop
    I = I[::2, ::2, 0]  # downsample by factor of 2
    I[I == 144] = 0  # erase background (background type 1)
    I[I == 109] = 0  # erase background (background type 2)
    I[I != 0] = 1  # everything else (paddles, ball) just set to 1
    return I.astype(np.float).ravel()

class PolicyNet(nn.Module):
    def __init__(self):
        super(PolicyNet, self).__init__()
        self.net = nn.Sequential(
            nn.Linear(D, H),
            nn.ReLU(inplace=True),
            nn.Linear(H, 2),
            nn.LogSoftmax()
        )
        if USE_CUDA:
            self.cuda()

    def forward(self, x):
        return self.net(x)

    def save(self, fname):
        torch.save(self.state_dict(), fname)

    def load(self, fname):
        self.load_state_dict(torch.load(fname))

class Policy:
    def __init__(self, cpdir, cpid):
        self.x0 = None
        self.net = PolicyNet()
        self.net.load(get_checkpoint_file(cpdir, cpid))
        self.rng = np.random.RandomState(42)

    def __call__(self, s):
        cur_x = prepro(s)
        x = cur_x - self.x0 if self.x0 is not None else np.zeros(D)
        self.x0 = cur_x

        # forward the policy network and sample an action from the returned probability
        x_tv = Variable(to_tensor_f32(x), requires_grad=False).unsqueeze(0)
        action_prob_log_tv = self.net.forward(x_tv)
        action_prob_log = to_numpy(action_prob_log_tv.data)[0]
        aprob = np.exp(action_prob_log)
        y = self.rng.choice(2, p=aprob)
        button_action = 2 if y == 0 else 3
        return button_action

def make_policy(cpdir, cpid):
    pol = Policy(cpdir, cpid)
    return pol


#!/usr/bin/env python
# coding: utf-8

# ## The following code does an analysis on stock price of Microsoft and makes a buying and seling decision based on 15ma and 200 #ma rule
# ### 1. Stock price of 5 years from 2015 to 2020 is fetched via Alpha vantage API
# ### 2. The buying decision is taken when the 15 ma signal crosses 200 ma signal from above
# ### 3. The selling action is taken when 15 ma signal crosses the 200 ma signal from below
# ### 4. Finding the final wealth of the trader, for both strategies, to an accuracy of two (2) decimal points
# ### 5. Plotting the outcome of the trader

# In[32]:

from alpha_vantage.timeseries import TimeSeries
import pandas as pd
import numpy as np
from alpha_vantage.techindicators import TechIndicators
import matplotlib.pyplot as plt

# In[33]:

api_key = '' # provide user api key
stock = 'MS'
start_dt = np.datetime64('2015-01-01')
end_dt = np.datetime64('2020-12-31')
long_MA = 200 #days
short_MA = 15 #days
initial_wealth = 1000

# ## Make use of the alpha vantage module to download the prices of stock from start_dt to end_dt.

# In[34]:

def slice_series(dt_frm, start_date, end_date):
#Return sliced time series based on the start ate and end date provided
    data_slice1 = dt_frm[dt_frm.index > start_date]
    data_slice2 = data_slice1[data_slice1.index <end_dt]
    return data_slice2

# In[46]:

ts = TimeSeries(key=api_key, output_format='pandas')
data, meta_data = ts.get_daily('MS', outputsize='full')
dt_data = slice_series(data,start_dt,end_dt)
print(f'The stock data of {stock} from {start_dt} to {end_dt} is as follows.')
dt_data

# ## Assumptions made:
#
# ### 1. The trading strategy is assumed to be without **human intervention**
#
# ### 2. The buying decision is taken when the 15 ma signal crosses 200 ma signal from above(sensing that the stock is undervalued now). If the price continues to fall, no discretion is made and the stock is hold.
#
# ### 3. The selling action is taken when 15 ma signal crosses the 200 ma signal from below(sensing that the stock is overvalued now)
#
# ### 4. The buy and sell will be done automatically even if the transaction ends in loss. No other strategies is considered
#

# In[36]:

ti = TechIndicators(key=api_key, output_format='pandas')
#downloading 15 ma
data_ti, meta_data_ti = ti.get_sma(symbol='MS', interval='daily',
                                    time_period=short_MA, series_type='close')
#downloading 200 ma
data_t2, meta_data_t2 = ti.get_sma(symbol='MS', interval='daily',
                                    time_period=long_MA, series_type='close')

# In[49]:

#slicing the data with the desired time period
dt_short_sma = slice_series(data_ti,start_dt,end_dt)
dt_long_sma = slice_series(data_t2,start_dt,end_dt)

#renaming the colums to plot
df1 = dt_short_sma
df1 = df1.rename(columns={'SMA': 'short ma'})
df2 = dt_long_sma
df2 = df2.rename(columns={'SMA': 'long ma'})

# ploting the trading signals
total_df = pd.concat([df2,df1], axis=1)
fig = plt.figure(figsize= [15,4])
total_df.plot()
plt.title('Trading Signal')
plt.show()

# In[44]:

#Adding closing price to the moving averages
total_df = pd.concat([dt_data['4. close'], df1,df2], axis=1)
total_df = total_df.rename(columns={'4. close': 'Closing price'})

#Adding signal to the series
len_total = len(total_df.index)
df_signal = total_df.assign(signal = np.zeros(len_total))
df_signal.loc[df_signal['short ma'] < df_signal['long ma'] , 'signal'] = 1

#creating buy signal
df_signal = df_signal.assign(buysignal = np.zeros(len_total))
df_signal = df_signal.assign(sellsignal = np.zeros(len_total))
for i in range(1,len_total) :
    df_signal.iloc[i,4] = df_signal.iloc[i, 3]-df_signal.iloc[i-1, 3]
df_signal.loc[df_signal['buysignal'] == -1, 'buysignal'] = 0

#creating sell signal
for i in range(len_total) :
    df_signal.iloc[i,5] = df_signal.iloc[i, 3]-df_signal.iloc[i-1, 3]
df_signal.loc[df_signal['sellsignal'] == 1, 'sellsignal'] = 0
df_signal.loc[df_signal['sellsignal'] == -1, 'sellsignal'] = 1

#finding buying and selling dates
sell_dates = df_signal[df_signal['sellsignal']==1].index
buy_dates = df_signal[df_signal['buysignal']==1].index
buy_sell_detail = pd.DataFrame()
buy_sell_detail = buy_sell_detail.assign(Buy_Date = buy_dates)
buy_sell_detail = buy_sell_detail.assign(Buy_Price = df_signal.loc[buy_dates]['Closing price'].tolist())
buy_sell_detail = buy_sell_detail.assign(Sell_Date = sell_dates)
buy_sell_detail = buy_sell_detail.assign(Sell_Price = df_signal.loc[sell_dates]['Closing price'].tolist())
print('The buy and sell dates and corresponding prices will be as shown below')
print(buy_sell_detail)

#
# ## Finding the final wealth of the trader, for both strategies, to an accuracy of two (2) decimal points

# In[39]:

#Creating BUY LOW and SELL HIGH dates
#initial wealth is $1000 till the first buy and sell happens.
df_signal = df_signal.assign(wealth_ma = np.zeros(len_total))
wealth_ma = initial_wealth
num_buy_sell = len(buy_sell_detail)

#intialising intial_wealth as accumulated wealth till first signal is observed
first_buy = buy_sell_detail['Buy_Date'][0]
df_signal.loc[df_signal.index < first_buy , 'wealth_ma'] = wealth_ma

#Calculating the wealth
for i in range(num_buy_sell):
    if(i>0):
        last_sell = buy_sell_detail['Sell_Date'][i-1]
        df_signal.loc[df_signal.index > last_sell , 'wealth_ma'] = wealth_ma
    buy_p = buy_sell_detail['Buy_Price'][i]
    units_p = wealth_ma/buy_p
    start_i=buy_sell_detail['Buy_Date'][i]
    end_i = buy_sell_detail['Sell_Date'][i]
    wealth_x = wealth_ma
    for x in df_signal[start_i:end_i].index:
        wealth_x = df_signal[df_signal.index == x]['Closing price'] * units_p
        df_signal.loc[df_signal.index == x , 'wealth_ma'] = wealth_x[0]
    wealth_ma = wealth_x[0]

#prefilling the last wealth accumulated till the end of period
last_sell = buy_sell_detail['Sell_Date'][num_buy_sell-1]
df_signal.loc[df_signal.index > last_sell , 'wealth_ma'] = wealth_ma

# In[40]:

#Creatiny BUY and HOLD strategy
#buy and hold
price_buy = total_df['Closing price'][0]
units = initial_wealth/price_buy
len_period = len(df_signal)
df_signal = df_signal.assign(wealth_hold = np.zeros(len_total))
df_signal['wealth_hold'] = units*total_df['Closing price']
final_wealth = df_signal['wealth_hold'][len_period-1]
print(f'The wealth accumulated by investing ${initial_wealth} and holding is ${final_wealth:.2f}.')
print(f'The wealth accumulated by investing ${initial_wealth} and using the moving avaerge buy/sell strategy is ${wealth_ma:.2f}.')

# ##  Plotting the outcome of the trader

# In[42]:

#Plotting the result
plt.clf()
df_signal[['wealth_ma','wealth_hold']].plot()
plt.show()


# -*- coding: UTF-8 -*-
# 这不是我的原创，不过我从这份源码中学习到了很多，感觉非常受益。
# 我认真读过以后，又加了一些注释、进行了一点修改。
# import csv
import numpy as np
from numpy import matrix
from math import pow
from collections import namedtuple
import math
import random
import os
import json

import copy
import scipy.signal as signal

def rand_arr(a, b, *args):
    np.random.seed(0)
    return np.random.rand(*args) * (b - a) + a

class CNN:
    # 保存神经网络的文件路径，我没有加保存功能
    # NN_FILE_PATH = 'CNN.json'
    def __init__(self, data_matrix, data_labels, training_indices):
        self.C_SIZE = 5  #卷积核/滤波器的长与宽，据说一般要是奇数？28*28的图像，按经验C_SIZE=5
        self.F_NUM = 12  #全连接层的卷积核/滤波器的长与宽，这里就不是奇数了
        self.P_SIZE = 2  #池化窗口的长与宽
        self.MAX_ITER_NUM = 10 #本来是50，很花时间；但是也不要改成1，准确率太低了

        self.data_matrix = data_matrix #卷积层输入图像
        self.data_labels = data_labels

        cLyNum = 20
        pLyNum = 20
        fLyNum = 100
        oLyNum = 10
        self.cLyNum = cLyNum  #卷积层输出、池化层输入图像的个数 = 卷积核数 = feature map数
        self.pLyNum = pLyNum  #池化层输出、全连接层输入图像的个数, = cLyNum
        self.fLyNum = fLyNum  #全连接层输出图像深度，全连接层输出图像为matrix 1*fLyNum
        self.oLyNum = oLyNum  #输出神经元数目，matrix 1*fLyNum -> 1*oLyNum
        self.pSize = self.P_SIZE
        self.yita = 0.01  #学习率

        self.cLyBias = rand_arr(-0.1, 0.1, 1,cLyNum) #matrix 1*cLyNum
        self.fLyBias = rand_arr(-0.1, 0.1, 1,fLyNum) #matrix 1*fLyNum
        self.kernel_c = np.zeros((self.C_SIZE,self.C_SIZE,cLyNum))
        self.kernel_f = np.zeros((self.F_NUM,self.F_NUM,fLyNum))
        for i in range(cLyNum):
        	#各个卷积核也是参数，它的初始化也是随机的，之后也通过反向传播来更新
            self.kernel_c[:,:,i] = rand_arr(-0.1,0.1,self.C_SIZE,self.C_SIZE)
        for i in range(fLyNum):
            #全连接层的卷积核
            self.kernel_f[:,:,i] = rand_arr(-0.1,0.1,self.F_NUM,self.F_NUM)
        self.pooling_a = np.ones((self.pSize,self.pSize))/(self.pSize**2)
        self.weight_f = rand_arr(-0.1,0.1, pLyNum, fLyNum)
        self.weight_output = rand_arr(-0.1,0.1,fLyNum,oLyNum)

        # 训练并保存
        TrainData = namedtuple('TrainData', ['y0', 'label'])
        for i in range(self.MAX_ITER_NUM):
            #注意，用CNN的话不要把图像的灰度值归一处理，效果很差
            self.train([TrainData(self.data_matrix[i], int(self.data_labels[i])) for i in training_indices])
        #self.save()

    def _rand_initialize_weights(self, size_in, size_out):
        return [((x * 0.12) - 0.06) for x in np.random.rand(size_out, size_in)]

    #卷积层
    def convolution(self, data, kernel):
        data_row, data_col = np.shape(data)
        kernel_row, kernel_col = np.shape(kernel)
        n = data_col - kernel_col
        m = data_row - kernel_row
        state = np.zeros((m+1, n+1))
        for i in range(m+1):
            for j in range(n+1):
                temp = np.multiply(data[i:i+kernel_row,j:j+kernel_col], kernel)
                state[i][j] = temp.sum()
        return state
    # 池化层（通过平均池化来压缩，不过一般选用最大池化）
    def pooling(self, data, pooling_a):
        data_r, data_c = np.shape(data)
        p_r, p_c = np.shape(pooling_a)
        r0 = data_r/p_r
        c0 = data_c/p_c
        state = np.zeros((r0,c0))
        for i in range(c0):
            for j in range(r0):
                temp = np.multiply(data[p_r*i:p_r*(i+1),p_c*j:p_c*(j+1)],pooling_a)
                state[i][j] = temp.sum()
        return state
    #全连接层，相当于乘以一个weight_f的共用参数，再进行卷积
    def convolution_f1(self, state_p1, kernel_f1, weight_f1):
        #池化层出来的20个特征矩阵乘以池化层与全连接层的连接权重进行相加
        #wx(这里的偏置项=0),这个结果然后再和全连接层中的神经元的核
        #进行卷积,返回值:
        #1：全连接层卷积前,只和weight_f1相加之后的矩阵
        #2：和全连接层卷积完之后的矩阵

        #cLyNum, fLyNum
        n_p0, n_f = np.shape(weight_f1)#n_p0=20(是Feature Map的个数);n_f是100(全连接层神经元个数)
        #m_p, n_p, cLyNum
        m_p, n_p, pCnt = np.shape(state_p1)#这个矩阵是三维的
        #F_NUM, F_NUM, fLyNum
        m_k_f1, n_k_f1,fCnt = np.shape(kernel_f1)#12*12*100

        state_f1_temp = np.zeros((m_p,n_p,n_f))
        state_f1 = np.zeros((m_p - m_k_f1 + 1,n_p - n_k_f1 + 1,n_f))
        # state_p(m_c*n_c*cLyNum) dot weight_f(cLyNum*fLyNum) = state_f1_temp(m_c*n_c*fLyNum)
        # 本来矩阵乘法中，还要在最外层对state_p1的前两维进行循环；
        for n in range(n_f):
            count = 0
            for m in range(n_p0):
                # 不过这里可以直接在循环内部进行向量乘法、求和
                temp = state_p1[:,:,m] * weight_f1[m][n]
                count = count + temp
            state_f1_temp[:,:,n] = count
            # 再对state_f1_temp做卷积
            state_f1[:,:,n] = self.convolution(state_f1_temp[:,:,n], kernel_f1[:,:,n])
        return state_f1, state_f1_temp
    # softmax 层
    def softmax_layer(self,state_f1):
        # print 'softmax_layer'
        output = np.zeros((1,self.oLyNum))
        t1 = (np.exp(np.dot(state_f1,self.weight_output))).sum()
        for i in range(self.oLyNum):
            t0 = np.exp(np.dot(state_f1,self.weight_output[:,i]))
            output[:,i]=t0/t1
        return output
    #误差反向传播更新权值，我还不是很懂
    def cnn_upweight(self,err_cost, ylab, train_data,state_c1, state_s1, state_f1, state_f1_temp, output):
        #print 'cnn_upweight'
        m_data, n_data = np.shape(train_data)
        # softmax的资料请查看 (TODO)
        label = np.zeros((1,self.oLyNum))
        label[:,ylab] = 1
        delta_layer_output = output - label  #最终输出层的误差（符号是负的，所以更新时是-=而不是+=）

        weight_output_temp = copy.deepcopy(self.weight_output)
        delta_weight_output_temp = np.zeros((self.fLyNum, self.oLyNum))
        #print np.shape(state_f1)
        #更新weight_output
        for n in range(self.oLyNum):
            # state_fo dot weight_output + bias 再激活-> layer_output
            # 故weight_output += yita * (delta_layer_output dot state_fo)
            delta_weight_output_temp[:,n] = delta_layer_output[:,n] * state_f1
        weight_output_temp = weight_output_temp - self.yita * delta_weight_output_temp

        #更新bias_f和kernel_f (推导公式请查看 TODO)
        delta_layer_f1 = np.zeros((1, self.fLyNum))
        delta_bias_f1 = np.zeros((1,self.fLyNum))
        delta_kernel_f1_temp = np.zeros(np.shape(state_f1_temp))
        kernel_f_temp = copy.deepcopy(self.kernel_f)
        for n in range(self.fLyNum):
            # 这层的误差（即delta_layer_f1）=
            # 下一层的误差（即delta_layer_output）
            # dot 这层的参数（即weight_output）
            # 整体 multiply 上层的激活函数的导数代入sum（即tanh^-1(state_f_pre)）
            count = 0
            for m in range(self.oLyNum):
                count = count + delta_layer_output[:,m] * self.weight_output[n,m]
            #本来是multiply，但是这里展开对每一维计算，每一维都是数*数，dot也可以
            delta_layer_f1[:,n] = np.dot(count, (1 - np.tanh(state_f1[:,n])**2))
            delta_bias_f1[:,n] = delta_layer_f1[:,n]
            delta_kernel_f1_temp[:,:,n] = delta_layer_f1[:,n] * state_f1_temp[:,:,n]
        # 1
        self.fLyBias = self.fLyBias - self.yita * delta_bias_f1
        kernel_f_temp = kernel_f_temp - self.yita * delta_kernel_f1_temp

        #更新weight_f1
        delta_layer_f1_temp = np.zeros((self.F_NUM,self.F_NUM,self.fLyNum))
        delta_weight_f1_temp = np.zeros(np.shape(self.weight_f))
        weight_f1_temp = copy.deepcopy(self.weight_f)
        for n in range(self.fLyNum):
            delta_layer_f1_temp[:,:,n] = delta_layer_f1[:,n] * self.kernel_f[:,:,n]
        for n in range(self.pLyNum):
            for m in range(self.fLyNum):
                temp = delta_layer_f1_temp[:,:,m] * state_s1[:,:,n]
                delta_weight_f1_temp[n,m] = temp.sum()
        weight_f1_temp = weight_f1_temp - self.yita * delta_weight_f1_temp

        # 更新bias_c1
        n_delta_c = m_data - self.C_SIZE + 1
        delta_layer_p = np.zeros((self.F_NUM,self.F_NUM,self.pLyNum))
        delta_layer_c = np.zeros((n_delta_c,n_delta_c,self.pLyNum))
        delta_bias_c = np.zeros((1,self.cLyNum))
        for n in range(self.pLyNum):
            count = 0
            for m in range(self.fLyNum):
                count = count + delta_layer_f1_temp[:,:,m] * self.weight_f[n,m]
            delta_layer_p[:,:,n] = count
            #print np.shape(np.kron(delta_layer_p[:,:,n], np.ones((2,2))/4))
            delta_layer_c[:,:,n] = np.kron(delta_layer_p[:,:,n], np.ones((2,2))/4) * (1 - np.tanh(state_c1[:,:,n])**2)
            delta_bias_c[:,n] = delta_layer_c[:,:,n].sum()
        # 2
        self.cLyBias = self.cLyBias - self.yita * delta_bias_c
        #更新 kernel_c1
        delta_kernel_c1_temp = np.zeros(np.shape(self.kernel_c))
        for n in range(self.cLyNum):
            temp = delta_layer_c[:,:,n]
            r1 = map(list,zip(*temp[::1]))#逆时针旋转90度
            r2 = map(list,zip(*r1[::1]))#再逆时针旋转90度
            temp = signal.convolve2d(train_data, r2,'valid')
            temp1 = map(list,zip(*temp[::1]))
            delta_kernel_c1_temp[:,:,n] = map(list,zip(*temp1[::1]))
        self.kernel_c = self.kernel_c - self.yita * delta_kernel_c1_temp
        self.weight_f = weight_f1_temp
        self.kernel_f = kernel_f_temp
        self.weight_output = weight_output_temp

    def train(self, training_data_array):
        for data_total in training_data_array:
            data = data_total.y0
            ylab = data_total.label

            self.predict(data, ylab)  #预测并训练

    def predict(self, test, ylab=None):
        data = test.reshape(28,28)
        d_m, d_n = np.shape(data)  #卷积层输入图像的长与宽
        # 假设输入图像为w1 * h1 * c1，w1表示宽度，h1表示高度，c1表示通道数（图像的深度），输出图像为w2 * h2 * c2
        # 在卷积层中，n表示卷积核的个数，k*k表示卷积核大小，p表示扩充边缘，s表示卷积核步长，那么有：
        # w2 = (w1 + 2 * p - k) / s + 1
        # h2 = (h1 + 2 * p - k) / s + 1
        # c2 = n
        # 这里做步长stride=1的卷积，扩充边缘p=0
        m_c = d_m - self.C_SIZE + 1  #卷积层输出、池化层输入图像的长与宽
        n_c = d_n - self.C_SIZE + 1  #d_m = d_n = 28, C_SIZE = 5, 28-5+1 = 24
        m_p = m_c/self.pSize #m_c = n_c = 24, pSize = 2, 24/2 = 12
        n_p = n_c/self.pSize
        state_c = np.zeros((m_c, n_c,self.cLyNum))  #卷积层输出、池化层输入图像的深度
        state_p = np.zeros((m_p, n_p, self.pLyNum))  #池化层输出图像的深度
        for n in range(self.cLyNum):
            state_c[:,:,n] = self.convolution(data, self.kernel_c[:,:,n])  #卷积
            tmp_bias = np.ones((m_c,n_c)) * self.cLyBias[:,n]  # 图像矩阵的每一位都加上偏置
            state_c[:,:,n] = np.tanh(state_c[:,:,n] + tmp_bias)# 加上偏置项然后过激活函数
            state_p[:,:,n] = self.pooling(state_c[:,:,n],self.pooling_a) #池化层输出压缩过的图像
        #全连接层
        #m_p = n_p = 12, F_NUM = 12, 12-12+1 = 1
        #state_f为1*1*fLyNum，state_f_pre为m_c*n_c*fLyNum
        # 变成1维才能作为普通神经网络的输入，进入softmax层
        state_f, state_f_pre = self.convolution_f1(state_p,self.kernel_f, self.weight_f)
        #进入激活函数
        state_fo = np.zeros((1,self.fLyNum))#全连接层经过激活函数的结果
        for n in range(self.fLyNum):
            state_fo[:,n] = np.tanh(state_f[:,:,n] + self.fLyBias[:,n])
        #进入softmax层
        output = self.softmax_layer(state_fo)
        #计算误差
        y_pre = output.argmax(axis=1)

        # 如果预测完了还要进行训练
        if ylab != None:
            err = -output[:,ylab]
            #print output
            #计算误差
            #print err
            self.cnn_upweight(err,ylab,data,state_c,state_p, state_fo, state_f_pre, output)
            # print self.kernel_c
            # print self.cLyBias
            # print self.weight_f
            # print self.kernel_f
            # print self.fLyBias
            # print self.weight_output
        return y_pre


from os import path
import sys

import django

from django.conf import settings

if not settings.configured:
    settings_dict = dict(
        INSTALLED_APPS=(
            'django_record',
            'django_record.tests'
        ),
        DATABASES={
            "default": {
                "ENGINE": "django.db.backends.sqlite3",
            }
        },
    )

    # configure settings
    settings.configure(**settings_dict)
    if django.VERSION >= (1, 7):
        django.setup()

def runtests():
    sys.path.insert(0, path.dirname(path.abspath(__file__)))

    # test for django under 1.8
    try:
        from django.test.simple import DjangoTestSuiteRunner
        runner_class = DjangoTestSuiteRunner
        test_args = ['tests']

    # test for django 1.8+
    except ImportError:
        from django.test.runner import DiscoverRunner
        runner_class = DiscoverRunner
        test_args = ['django_record']

    # run tests
    failures = runner_class(
        verbosity=2, interactive=True, failfast=False
    ).run_tests(test_args)

    # exit with errors
    sys.exit(failures)

if __name__ == '__main__':
    runtests()


#!/usr/bin/env python

import os
import sys
import json
import pylab
import numpy as np
import matplotlib.ticker
from dateutil import parser
from scipy import interpolate
from matplotlib.dates import date2num
from datetime import timedelta, datetime
from pysurvey.plot import setup, dateticks, minmax, hcolorbar

FILENAME = os.path.expanduser('~/.temperature.neon.log')
SEP = ' : '
YR = [30,75]
TNORM = 38

FILENAME = os.path.expanduser('~/.temperature.xenon.log')
SEP = ' | '
YR = [5,60]
TNORM = 25

def read_temps(filename=FILENAME):
    out = []
    with open(filename, 'r') as f:
        lines = f.readlines()
        for line in lines:
            items = line.split(SEP)
            out.append(dict(date=parser.parse(items[1]),
                            temperature=float(items[2])/1000.0))
    return out

def setupplot(secondax=False, **kwargs):
    ytickv = np.linspace(YR[0],YR[1],6)
    yticknames = map('{:0.0f}'.format, ytickv)
    tmp = dict(
        ylabel='Temperature [c]',
        yr=minmax(ytickv), ytickv=ytickv,
        yticknames=yticknames,
    )
    tmp.update(kwargs)
    ax = setup(**tmp)

    if secondax:
        subplt = kwargs.get('subplt',None)
        f = lambda x: '{:0.0f}'.format(1.8*x + 32.0)
        yticknames = map(f, ytickv)
        ax2 = ax.twinx()
        ax2.set_ylabel(r"Temperature [F]")
        ax2.set_ylim(minmax(ytickv))
        ax2.yaxis.set_major_locator(matplotlib.ticker.FixedLocator(ytickv))
        ax2.yaxis.set_major_formatter(matplotlib.ticker.FixedFormatter(yticknames))
        pylab.sca(ax)

        # setup(ax=ax.twinx(),
        #       subplt=subplt,
        #       ylabel='Temperature [F]',
        #       yr=minmax(ytickv), ytickv=ytickv, yticknames=yticknames)

    return ax

def get_continuum(dates, x, y, delta=2):
    out = []
    t = timedelta(hours=delta)
    for d in dates:
        ii = np.where( (date2num(x) >  date2num(d-t) ) &
                       (date2num(x) <= date2num(d+t) ) )[0]
        if len(ii) <= 20:
            out.append(-1)
        else:
            out.append( np.mean(y[ii]) )

    t = date2num(dates)
    n = np.min(t)
    tmp = np.array(out)
    ii = np.where(tmp > 0)
    f = interpolate.UnivariateSpline(date2num(dates[ii])-n,tmp[ii], s=3)
    out = f(t-n)
    return out

def plot_weather(mindate):
    weatherfile = os.path.expanduser('~/data/weather.json')
    wf = json.load(open(weatherfile,'r'))
    keys = sorted(wf.keys())
    x = np.array(map(np.float, keys))
    y = np.array(map(np.float, [wf[k]['tempm'] for k in keys]))+50
    ii = np.where(x > mindate)
    pylab.plot(x[ii],y[ii])

def plot_temp():
    data = read_temps()
    dates, values = map(np.array, zip(*[(d['date'], d['temperature'])
                                        for d in data]))
    tmp = (date2num(dates) % 1.0)*24.0
    ii = np.where((tmp > 0) & (tmp < 8))[0]
    continuum = get_continuum(dates, dates[ii], values[ii])

    setup(figsize=(12,6))

    setupplot(subplt=(1,2,1), autoticks=True, xlabel='Date',)
    pylab.plot(dates, values)
    pylab.plot(dates[ii], values[ii], '.r')
    pylab.plot(dates, continuum, '.k')
    plot_weather(np.min(date2num(dates)))
    # pylab.plot(dates, values-continuum+38, '.r')
    dateticks('%Y.%m.%d')

    setupplot(subplt=(2,2,2), autoticks=False, xlabel='Hour of Day')
    pylab.plot(tmp, values, '.')
    setupplot(subplt=(2,2,2), ylabel='', secondax=True)

    setupplot(subplt=(2,2,4), autoticks=False, xlabel='Hour of Day')
    sc = pylab.scatter(tmp, values-continuum+TNORM,
                       c=date2num(dates)-np.min(date2num(dates)), s=15,
                       marker='.', edgecolor='none',
                       label='Days since Start')

    setupplot(subplt=(2,2,4), ylabel='', secondax=True)
    hcolorbar(sc, axes=[0.75, 0.42, 0.1, 0.01])

    pylab.tight_layout()
    pylab.show()

if __name__ == '__main__':
    from pysurvey import util
    util.setup_stop()
    plot_temp()


import math

import torch

from torch.nn.parameter import Parameter
from torch.nn.modules.module import Module

class GraphConvolution(Module):
    """
    Simple GCN layer, similar to https://arxiv.org/abs/1609.02907
    """

    def __init__(self, in_features, out_features, adj, bias=False, activation=None):
        super(GraphConvolution, self).__init__()
        self.in_features = in_features
        self.out_features = out_features
        self.adj = adj
        # self.adj = adj.to_dense()
        self.weight = Parameter(torch.FloatTensor(in_features, out_features))
        if bias:
            self.bias = Parameter(torch.FloatTensor(out_features))
        else:
            self.register_parameter('bias', None)
        self.reset_parameters()
        self.activation = activation

    def reset_parameters(self):
        stdv = 1. / math.sqrt(self.weight.size(1))
        self.weight.data.uniform_(-stdv, stdv)
        if self.bias is not None:
            self.bias.data.uniform_(-stdv, stdv)

    # def forward(self, input, adj):
    def forward(self, input):
        support = torch.mm(input, self.weight)
        #output = torch.spmm(self.adj, support)
        output = torch.mm(self.adj, support)
        if self.bias is not None:
            pre_ac = output + self.bias
        else:
            pre_ac = output
        if self.activation:
            return self.activation(pre_ac)
        else:
            return pre_ac

    def __repr__(self):
        return self.__class__.__name__ + ' (' \
               + str(self.in_features) + ' -> ' \
               + str(self.out_features) + ')'


"""empty message

Revision ID: 5fe110ca29e7
Revises: 04e6148ab996
Create Date: 2018-06-18 17:42:59.740765

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql

# revision identifiers, used by Alembic.
revision = '5fe110ca29e7'
down_revision = '04e6148ab996'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('Request', sa.Column('type_id', postgresql.UUID(as_uuid=True), nullable=False))
    op.drop_constraint('Request_request_id_fkey', 'Request', type_='foreignkey')
    op.create_foreign_key(None, 'Request', 'Request_type', ['type_id'], ['id'])
    op.drop_column('Request', 'request_id')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('Request', sa.Column('request_id', postgresql.UUID(), autoincrement=False, nullable=False))
    op.drop_constraint(None, 'Request', type_='foreignkey')
    op.create_foreign_key('Request_request_id_fkey', 'Request', 'Request_type', ['request_id'], ['id'])
    op.drop_column('Request', 'type_id')
    # ### end Alembic commands ###


from predict import Predict
from sys import exit
from args import *
from db import *
import json

data = {
	"type" : None,
	"ratio" : {},
	"top" : []
}

try:
	cnx = connect(**base)
	svms, ngrams = select_last_weights(cnx)
	indexes = select_indexes(cnx)

	predict = Predict(svms, indexes, ngrams)
	uid = predict.text([text]).pop()
	predict.calc()

	data["type"] = predict.type(uid)
	data["ratio"] = predict.ratio(uid)
	data["top"] = predict.top()
except:
	pass
finally:
	data = json.dumps(data)
	print(data)
	exit(0)


from tkinter import messagebox
import sqlite3
from tkinter import simpledialog
from tkinter.constants import END
from FuncionesCRUD_POO import *

def conectarBBDD():

    nombre_BBDD=simpledialog.askstring("BBDD", "Introduce el nombre de la BBDD")

    almacena(nombre_BBDD)

    miConexion=sqlite3.connect(nombre_BBDD)

    miCursor=miConexion.cursor()

    try:
        miCursor.execute('''

            CREATE TABLE DATOSUSUARIOS(
                ID INTEGER PRIMARY KEY AUTOINCREMENT,
                NOMBRE VARCHAR(50),
                PASSWORD VARCHAR(50),
                APELLIDO VARCHAR(50),
                DIRECCION VARCHAR (70),
                COMENTARIOS VARCHAR (200)
            )
        ''')

        messagebox.showinfo("BBDD", "Base de Datos creada con exito")

    except:

        messagebox.showwarning("Cuidao", "La Base de Datos ya existe")

def salirAplicacion(raiz):

    valor_salir=messagebox.askquestion("Salir", "Deseas salir de la aplicacion?")

    if valor_salir=="yes":
        raiz.destroy()

# def limpiarCampos(campo1, campo2, campo3, campo4, campo5):
#     campo1.set("")
#     campo2.set("")
#     campo3.set("")
#     campo4.set("")
#     campo5.set("")

def limpiarCampos(*args):

    for campo in args:

        campo.delete(1.0, END)

    else:

        campo.set("")

    # cuadroTexto.delete(1.0, END)


from datetime import date
from pytest import mark
from definitions import SpacetimePoint
from gateways import FileGateway

@mark.parametrize('filename', [
    '123432_2018-09-03_json.csv',
    '234234_2017-12-01_json.csv'
    ])
def test_identifies_valid_files(filename):
    assert FileGateway._valid_file(filename)

@mark.parametrize('filename', [
    'NA_2016-12-11_json.csv',
    'cancelled_2015-01-01_json.csv'
    ])
def test_identifies_invalid_files(filename):
    assert not FileGateway._valid_file(filename)

def test_makes_filename():
    point = SpacetimePoint(123.43, -54.345, date(2018, 1, 1))
    file_path = '/home/gklatte'
    filename = FileGateway._create_filename(point, file_path)
    expected_filename = '/home/gklatte/123.43_-54.345_2018-01-01.json'

    assert expected_filename == filename


import sqlalchemy as sql
import pandas as pd
from datetime import datetime
engine = sql.create_engine('mysql+pymysql://simplon:simplon@localhost/ASSURAUTO')

CL_ID = pd.read_sql_query('SELECT max(CL_ID) as max FROM CLIENTS;' , engine)['max'].values
CL_ID = CL_ID[0]+1
CO_CLIENTS_FK = CL_ID
CL_PRENOM = input('Entrez le prenom : ')
CL_NOM = input('Entrez le nom : ').upper()
CL_ADRESSE = input("Entrez l'adresse : ")
CL_POSTALE =''
while len(CL_POSTALE) == 0 :
    CL_POSTALE = input('Entrez le code postale : ')

    if (CL_POSTALE.isdigit()):
        CL_POSTALE = CL_POSTALE

    else:
        print("ERRROR")
        CL_POSTALE = ''

CL_VILLE = input('Entrez la ville : ')
CL_COORDONNEES = input('Entrez les coordonnees : ')

engine.execute('INSERT INTO CLIENTS (CL_ID, CL_PRENOM, CL_NOM, CL_ADRESSE, CL_POSTALE, CL_VILLE, CL_COORDONNEES) values (%s,"%s","%s","%s",%s,"%s",%s);' %(CL_ID, CL_PRENOM, CL_NOM, CL_ADRESSE, CL_POSTALE, CL_VILLE, CL_COORDONNEES))

CO_CATEGORIE = input('Entrez la categorie : ')
CO_BONUS_MALUS = input('Entrez le bonus/malus : ')

engine.execute('INSERT INTO CONTRAT (CO_NUM, CO_DATE, CO_CATEGORIE, CO_BONUS_MALUS, CO_CLIENTS_FK) values ((SELECT max(CO_NUM)+1 from CONTRAT as CO), CURDATE(),"%s",%s,%s);' %( CO_CATEGORIE, CO_BONUS_MALUS,CO_CLIENTS_FK))


import discord
from discord.ext import commands
from cogs.utils import checks
from datetime import datetime, timedelta
import re
from langdetect import detect

class langdetect:
    """Shows how much french vs english is shown in a channel"""

    def __init__(self, bot):
        self.bot = bot

    @commands.command(pass_context=True)
    @checks.is_owner()
    async def langdetect(self, ctx, channel, threshold):
        """Simply give the channel id and how many days old the messages can be"""

        if not threshold.isdigit(): # Checks wether the threshold given is a number
            await self.bot.say("Threshold should be numbers. Aborting.")
            return

        msg = await self.bot.say("Processing please wait...")
        await self.get_messages(ctx.message.server, channel, int(threshold))
        await self.bot.delete_message(msg)

    async def get_messages(self, server, channel, threshold): # Gets messages and detects the language
        after_n_days_ago = datetime.utcnow() - timedelta(days=threshold)
        channel_id = self.bot.get_channel(str(channel))
        frcounter = 0
        encounter = 0
        async for mesg in self.bot.logs_from(channel_id, after=after_n_days_ago, limit=100*100, reverse=True):
            mesg = mesg.clean_content
            cleantext = self.letters(mesg)
            if cleantext == "" or cleantext == None or type(cleantext) != str or cleantext.isspace() == True:
                pass
            else:
                if detect(cleantext) == 'fr':
                    frcounter = frcounter + 1
                if detect(cleantext) == 'en':
                    encounter = encounter + 1
        await self.bot.say("French : {} \nEnglish : {}".format(frcounter, encounter))

    def letters(self, mesg): # Cleans messages to remove non_alphanumeric characters
        valids = []
        for character in mesg:
            if character.isspace():
                valids.append(character)
            if character.isalpha():
                valids.append(character)
        cleantext = ''.join(valids)
        return cleantext

def setup(bot):
    bot.add_cog(langdetect(bot))


import contextlib
import json
import logging
import os
import random
from collections import OrderedDict
from functools import partial

import numpy as np
import tensorflow as tf
from tensorflow.keras.preprocessing import image
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.preprocessing.text import Tokenizer

class DataLoader():
    """
    A class used to package related data processing constants and methods
    for training and executing an encoder-decoder model. It can be initialized with training=False
    to simpy get the initialized constants and word_list.json used for
    beam_search in the caption module.

    Attributes
    ----------
    WORD_EMBED_SIZE : int
        size of word embedding in decoder model
    IMAGE_SIZE : int
        size of length and width dimension for encoder input
    DATA_GEN_ARGS : dict
        a dictionary of image transformations to pass to ImageDataGenerator
    AUTOTUNE : obj
        lazy declaration of parameters and optimizes dataprocessing parameters
        when datasets are consumed, i.e. PARALLEL_CALLS and BUFFER_SIZE
    PARALLEL_CALLS : int
        number of paralllel calls to make when mapping over a batch
    BUFFER_SIZE: int
        number of batches to prefetch
    TRAIN_P, VAL_P, TEST_P: int
        percentage splits of dataset, should all add up to 1

    Methods
    -------
    get_train_dataset, get_val_dataset, get_test_dataset
        returns dataset to be consumed by training, validating, and testing
        processes respectively in ModelTrainer class

    batch_convert_to_words(captions, references=True)
        converts references and hypotheses from int sequences to words

    batch_sample_target(target)
        selects random caption from each list in target

    load(img, path=False)
        loads image from image_id or path img and resizes and converts to array

    preprocess_input(x), unprocess_input(x)
        transforms and reverses transform image tensor/array x respectively
        using transformations expected by encoder

    create_word_embedding_matrix
        creates glove embeddings of size WORD_EMBED_SIZE, must download correct
        glove embedding file to resize to other dimensions
    """

    WORD_EMBED_SIZE = 300
    IMAGE_SIZE = 299
    DATA_GEN_ARGS = dict(rotation_range=45,
                         shear_range=0.20,
                         width_shift_range=0.2,
                         height_shift_range=0.2,
                         zoom_range=0.3,
                         horizontal_flip=True,
                         vertical_flip=False,
                         )
    AUTOTUNE = tf.data.experimental.AUTOTUNE
    PARALLEL_CALLS = AUTOTUNE
    BUFFER_SIZE = AUTOTUNE
    TRAIN_P, VAL_P, TEST_P = 0.9, 0.05, 0.05

    WORDS_PATH = f"{os.path.abspath('.')}/training/data/word_list.json"
    ANNOTATIONS_PATH = f"{os.path.abspath('.')}/training/data/annotations"
    IMAGES_PATH = f"{os.path.abspath('.')}/training/data/images"
    EMBEDDINGS_PATH = f"{os.path.abspath('.')}/training/data/glove.42B.300d.txt"

    logger = logging.getLogger('__main__.data_loading')

    def __init__(self, batch_size=32, vocab_size=10240, shuffle_size=8000, training=True):
        """
        Parameters
        ----------
        batch_size : int, optional
            number of image/caption pairs to batch from dataset; reduce if oom
        vocab_size : int, optional
            number of words to use in model from original MSCOCO corpus
            gathered from captions
        shuffle_size : int, optional
            how big the shuffle window is when shuffling the dataset
        training : bool
            whether DataLoader is going to be used to train or to help
            execute models; if train=False non-class methods should not be used
        """
        assert (vocab_size > 3 and
                vocab_size < 28040), "vocab_size not in range"
        self.vocab_size = vocab_size
        if (not os.path.exists(self.WORDS_PATH) or training):
            coco_train_file = f'{self.ANNOTATIONS_PATH}/captions_train2017.json'
            coco_val_file = f'{self.ANNOTATIONS_PATH}/captions_val2017.json'
            coco_files = [coco_train_file, coco_val_file]

            img_to_caps = OrderedDict()
            logger.info("Processing json data")
            with contextlib.ExitStack() as stack:
                files = [stack.enter_context(open(fname, 'r'))
                         for fname in coco_files]
                for f in files:
                    self._coco(f, img_to_caps)

            self.tokenizer = Tokenizer(num_words=self.vocab_size,
                                       oov_token="<unk>",
                                       filters='!"#$%&()*+,-./:;=?@[\\]^_`{|}~\t\n')
            logger.info("Fitting tokenizer on corpus")
            self.tokenizer.fit_on_texts(tf.nest.flatten(img_to_caps.values()))
            self.tokenizer.index_word[0] = '<pad>'
            self.tokenizer.word_index['<pad>'] = 0

            if not os.path.exists(self.WORDS_PATH):
                self._create_word_list(self.WORDS_PATH)

        if training:
            # set instance variables
            self.DATASET_SIZE = len(img_to_caps.values())
            self.batch_size = batch_size
            self.shuffle_size = shuffle_size

            self.image_datagen = ImageDataGenerator(**self.DATA_GEN_ARGS)

            # set functions
            self.batch_load = partial(tf.map_fn, self.load, dtype='float32')
            self.batch_transform = partial(tf.map_fn, self._transform,
                                           dtype='float32')
            self.batch_preprocess = partial(tf.map_fn, self._preprocess,
                                            dtype='float32')
            # captions list
            captions = self._process_captions(list(img_to_caps.values()))
            # data splits
            self.train, self.val, self.test = self._split_data(list(
                img_to_caps.keys()), captions)

    def _create_word_list(self, path):
        # create word_list at indicated path
        logger.info(f"Creating word_list.json at {WORDS_PATH}")
        data = []
        for i in range(self.vocab_size + 1):
            data.append(self.tokenizer.index_word[i])
        with open(path, 'w') as p:
            json.dump(data, p, indent=4)

    def _train_mapper(self, paths, seq_lists):
        # wrapper over _map_train_batch to map over tensors from train dataset
        return tf.numpy_function(self._map_train_batch, [paths, seq_lists],
                                 [tf.float32, tf.int32])

    def _validate_mapper(self, paths, seqs):
        # wrapper over _map_validate_batch to map over tensors from val dataset
        return tf.numpy_function(self._map_validate_batch, [paths, seqs],
                                 [tf.float32, tf.int32])

    def _transform(self, img):
        # wrapper over random_transform to be used in tensor mapping
        return tf.numpy_function(self.image_datagen.random_transform,
                                 [img], [tf.float32])

    def _preprocess(self, img):
        # wrapper over preprocess_input to be used in tensor mapping
        return tf.numpy_function(self.preprocess_input, [img],
                                 [tf.float32])

    def _map_train_batch(self, paths, captions):
        # core map function over train batches
        imgs = self.batch_load(paths)
        imgs = self.batch_transform(imgs)
        imgs = self.batch_preprocess(imgs)
        captions = self.batch_sample_target(captions)
        return imgs, captions

    def _map_validate_batch(self, paths, captions):
        # core map function over validate batches
        imgs = self.batch_load(paths)
        imgs = self.batch_preprocess(imgs)
        return imgs, captions

    def _split_data(self, x_list, y_list):
        # combines each member from x_list and y_list into an (x, y)
        # tuple and splits the resulting list into train, validate, and test
        # sublists to be later converted into datasets
        logger.info("Splitting data into train, val, and test datasets")
        train_split = int(self.TRAIN_P * self.DATASET_SIZE)
        val_split = int((self.TRAIN_P + self.VAL_P) * self.DATASET_SIZE)

        dataset_list = list(zip(x_list, y_list))
        # don't shuffle data before split, need inherent ordering
        # to train after restarting DataLoader
        train, validate, test = np.split(dataset_list,
                                         [train_split, val_split])
        return train, validate, test

    def _make_datasets(self, xy_list):
        # converts a list of (x, y) tuples into two separate datasets of
        # x and y respectively
        images = [x for x, y in xy_list]
        captions = [y for x, y in xy_list]
        images_ds = tf.data.Dataset.from_tensor_slices(
            (tf.convert_to_tensor(images, dtype='int32')))
        captions_ds = tf.data.Dataset.from_tensor_slices(
            (tf.convert_to_tensor(captions, dtype='int32')))
        return images_ds, captions_ds

    def _coco(self, json_file, d):
        # load json data into ordered dictionary of image_id : captions pairs
        annotations = json.load(json_file)
        for annot in annotations['annotations']:
            path = int(annot['image_id'])
            l = d.get(path, [])
            l.append(f"<start> {annot['caption']} <end>")
            d[path] = l

    def _process_captions(self, captions):
        # preprocesses captions by tokenizing, cropping, padding, and removing
        # any caption list with more than 5 captions, prefering to remove
        # shorter captions
        logger.info("Processing captions data")
        min_len, max_len = 10, 18
        end = self.tokenizer.word_index['<end>']

        # return False if sequence is not of acceptable length
        def F(s): return min_len <= len(s) <= max_len

        # pad sequences up to max length with <pad>
        def P(s): return tf.keras.preprocessing.sequence.pad_sequences(
            s, padding='post', maxlen=max_len)

        # return number of sequences over max allowed of 5
        def R(s): return len(s) - 5

        # returns indices of R sequences with least number of words
        def Z(captions):
            over = len(captions) - 5
            size_list = [len(c) for c in captions]
            res = sorted(range(len(size_list)),
                         key=lambda sub: size_list[sub])[:over]
            return res

        # tokenize captions
        captions = [self.tokenizer.texts_to_sequences(
            cap_list) for cap_list in captions]

        # crop sequences of incorrect length
        captions = [[cap if F(cap) else cap[:max_len - 1] + [end]
                     for cap in cap_list] for cap_list in captions]

        # remove cap_lists that have length > 5
        captions = [[c for i, c in enumerate(c_l) if i not in Z(
            c_l)] if R(c_l) else c_l for c_l in captions]

        # pad sequences to length max-length
        captions = [P(cap_list) for cap_list in captions]

        return captions

    def get_train_dataset(self):
        """Returns training dataset optimized to be consumed batchwise over
        many epochs
        """
        train_dataset = (
            tf.data.Dataset.zip((self._make_datasets(self.train)))
            .cache()
            .shuffle(self.shuffle_size)
            .batch(self.batch_size, drop_remainder=True)
            .map(self._train_mapper, num_parallel_calls=self.PARALLEL_CALLS)
            .prefetch(self.BUFFER_SIZE))

        return train_dataset

    def get_val_dataset(self):
        """Returns validation dataset optimized to be consumed batchwise over
        many epochs
        """
        val_dataset = (
            tf.data.Dataset.zip((self._make_datasets(self.val)))
            .cache()
            .shuffle(self.shuffle_size)
            .batch(self.batch_size, drop_remainder=True)
            .map(self._validate_mapper, num_parallel_calls=self.PARALLEL_CALLS)
            .prefetch(self.BUFFER_SIZE)
        )
        return val_dataset

    def get_test_dataset(self):
        """Returns test dataset optimized to be consumed batchwise over
        a single epoch
        """
        test_dataset = (
            tf.data.Dataset.zip((self._make_datasets(self.test)))
            .batch(self.batch_size, drop_remainder=True)
            .map(self._validate_mapper, num_parallel_calls=self.PARALLEL_CALLS)
            .prefetch(self.BUFFER_SIZE)
        )
        return test_dataset

    def batch_convert_to_words(self, captions, references=True):
        """Converts integer tensor to array of strings with same nested
        structure except not a tensor and converted from int32 to tokens from
        tokenizer with <start> and <end> tokens removed

        Parameters
        ----------
        captions : tensor of int
            either a tensor of shape (batch, list_size, padded_sequence_length)
            for references or (batch, padded_sequence_length) for hypotheses"""
        captions = captions.numpy()
        # tokenize sequences
        T = self.tokenizer.sequences_to_texts
        end = self.tokenizer.word_index['<end>']
        # find where the sequence ends with <end> token
        def E(caption): return np.where(caption == end)[0][0]

        sentences = []
        if references:
            filtered = [[sentence[1:E(sentence)]
                         for sentence in lst] for lst in captions]
            sentences = [T(lst) for lst in filtered]
        else:
            filtered = [sentence[1:E(sentence)] for sentence in captions]
            sentences = T(filtered)
        return sentences

    @classmethod
    def batch_sample_target(cls, target):
        """Returns a tensor of randomly selected sequences, from each list, of
        size (batch, padded_sequence_length)

        Parameters
        ----------
        target : int tensor of shape (batch, list_size, padded_sequence_length)
        """
        list_size = target.shape[1]  # should be 5
        return tf.cast([sample[np.random.randint(list_size)] for
                        sample in target], dtype='int32')

    @classmethod
    def load(cls, img, path=False):
        """Loads image and resizes / converts to array

        Parameters
        ----------
        img : str -- image_id or path to image, specified with path parameter
        path : bool -- indicates whether img is a path to an image"""
        if not path:
            img = f"{cls.IMAGES_PATH}/{img:012d}.jpg"
        img = image.load_img(img, target_size=(cls.IMAGE_SIZE, cls.IMAGE_SIZE))
        img = image.img_to_array(img)
        return img

    @classmethod
    def preprocess_input(cls, x):
        """transforms image tensor/array x using transformations expected by
        encoder

        Parameters
        ----------
        x : image array/tensor"""
        x /= 255.
        x -= 0.5
        x *= 2.
        return x

    @classmethod
    def unprocess_input(cls, x):
        """undoes transform image tensor/array x using transformations expected by
        encoder

        Parameters
        ----------
        x : transformed image array/tensor"""
        x /= 2.
        x += 0.5
        x *= 255.
        return x

    def create_word_embedding_matrix(self):
        """creates glove embeddings of size WORD_EMBED_SIZE, must download
        correct glove embedding file to resize to other dimensions"""

        logger.info(f"Creating word_embedding_matrix from {EMBEDDINGS_PATH}")
        word_embed_dict = dict()

        with open(self.EMBEDDINGS_PATH, 'r') as file:
            for line in file:
                values = line.split()
                word = values[0]
                # only map words that are in model vocabulary
                if self.tokenizer.word_index.get(word, self.vocab_size + 1) <= self.vocab_size:
                    embedding = np.asarray(values[1:], dtype='float32')
                    word_embed_dict[word] = embedding

        index_embed_matrix = np.zeros(
            (self.vocab_size + 1, self.WORD_EMBED_SIZE))
        for i in range(self.vocab_size + 1):
            w = self.tokenizer.index_word[i]
            word_embedding = word_embed_dict.get(w)
            if word_embedding is not None:
                index_embed_matrix[i] = word_embedding

        return index_embed_matrix


from django.shortcuts import render, HttpResponse
from django.contrib.auth.mixins import LoginRequiredMixin
from django.db import transaction
from django.db.models import F
from django.views import View
from Personalwebsite import settings as sys
from django.core.mail import EmailMultiAlternatives
from django.contrib.auth.models import User
import json
from APP_Comment.models import *
from App_Blog.models import Article

class AddComment(LoginRequiredMixin, View):
    def get(self, request):
        return HttpResponse(json.dumps('fuck you!'))

    def post(self, request):
        if request.is_ajax():
            userId = request.POST['userId']
            articleId = request.POST['articleId']
            content = request.POST['content'][:-2]
            browserId = request.POST['browserId']
            article_obj = Article.objects.filter(id=articleId)
            response = {}
            try:
                with transaction.atomic():
                    Comment.objects.create(user_id=userId, articleId_id=articleId, body=content, browserId=browserId)
                    article_obj.update(commentCount=F('commentCount') + 1)
                response['successComment'] = True
            except Exception:
                response['successComment'] = False
            subject = '文章《%s》有新的评论' % Article.objects.get(id=articleId).title
            username = User.objects.get(id=userId).username
            send_content = """

%s说:%s

去看看

                            """ % (username, content, articleId)
            msg = EmailMultiAlternatives(subject, send_content, sys.EMAIL_HOST_USER, [sys.EMAIL_SELF_ATTR, ])
            msg.content_subtype = "html"
            msg.send()
            return HttpResponse(json.dumps(response))
        else:
            cnm = '您的本次请求存在违法信息，已被系统标记！'
            return HttpResponse(json.dumps(cnm))

class AddReply(LoginRequiredMixin, View):
    def post(self, request):
        if request.is_ajax():
            replyPk = request.POST['replyPk']
            userId = request.user.pk
            articleId = request.POST['articleId']
            content = request.POST['content'][:-2]
            browserId = request.POST['browserId']
            status = {}
            if all([replyPk, userId, articleId, content, browserId]):
                article_obj = Article.objects.filter(id=articleId)
                # 查询到当前被评论的这条评论
                replyObj = Comment.objects.filter(id=replyPk).first()
                # 获取到他的根评论id
                try:
                    rootId = replyObj.root.id
                except:
                    rootId = replyPk
                parentId = replyPk
                # 获取到该条评论的用户名
                rootUser = replyObj.user.username
                rootUserPk = replyObj.user.id
                # 开启事物，同步数据
                try:
                    with transaction.atomic():
                        Comment.objects.create(user_id=userId,
                                               replayTo_id=rootUserPk,
                                               root_id=rootId,
                                               parentId_id=parentId,
                                               articleId_id=articleId,
                                               body=content,
                                               browserId=browserId)
                        article_obj.update(commentCount=F('commentCount') + 1)
                    status['success'] = True
                except:
                    status['success'] = False
                    return HttpResponse(json.dumps(status))
                # 构建出返回前端的数据
                status['replyToUser'] = rootUser
                status['rootId'] = rootId
            else:
                status['success'] = False
            return HttpResponse(json.dumps(status))
        else:
            cnm = '您的本次操作属于违规操作，已被系统标记。'
            return HttpResponse(json.dumps(cnm))


import shutil
import tempfile
from typing import Any, Dict, cast

import importlib_resources

from gt4sd.training_pipelines import (
    TRAINING_PIPELINE_MAPPING,
    PaccMannVAETrainingPipeline,
)

template_config = {
    "model_args": {
        "n_layers": 1,
        "bidirectional": False,
        "rnn_cell_size": 64,
        "latent_dim": 32,
        "stack_width": 8,
        "stack_depth": 8,
        "decoder_search": "sampling",
        "dropout": 0.2,
        "generate_len": 50,
        "kl_growth": 0.003,
        "input_keep": 0.85,
        "test_input_keep": 1.0,
        "temperature": 0.8,
        "embedding": "one_hot",
        "vocab_size": 380,
        "pad_index": 0,
        "embedding_size": 380,
    },
    "dataset_args": {
        "add_start_stop_token": True,
        "selfies": True,
        "num_workers": 1,
        "pin_memory": False,
    },
    "training_args": {
        "epochs": 1,
        "batch_size": 4,
        "learning_rate": 0.0005,
        "optimizer": "adam",
        "log_interval": 2,
        "save_interval": 2,
        "eval_interval": 2,
        "model_path": "/tmp/paccmann_vae",
        "training_name": "paccmann-vae-test",
    },
}

def test_train():

    pipeline = TRAINING_PIPELINE_MAPPING.get("paccmann-vae-trainer")

    assert pipeline is not None

    TEMPORARY_DIRECTORY = tempfile.mkdtemp()

    test_pipeline = cast(PaccMannVAETrainingPipeline, pipeline())

    config: Dict[str, Any] = template_config.copy()

    config["training_args"]["model_path"] = TEMPORARY_DIRECTORY
    with importlib_resources.as_file(
        importlib_resources.files("gt4sd") / "training_pipelines/tests/molecules.smi",
    ) as file_path:

        config["dataset_args"]["train_smiles_filepath"] = file_path
        config["dataset_args"]["test_smiles_filepath"] = file_path
        test_pipeline.train(**config)

        shutil.rmtree(TEMPORARY_DIRECTORY)


import os
import sys
from collections.abc import Sequence
from glob import glob
from os.path import dirname, realpath

from ..utils import guess_inkscape_config_path
from .palette import ThreadPalette

class _ThreadCatalog(Sequence):
    """Holds a set of ThreadPalettes."""

    def __init__(self):
        self.palettes = []
        self.load_palettes(self.get_palettes_paths())

    def get_palettes_paths(self):
        """Creates a list containing the path of two directories:
        1. Palette directory of Inkscape
        2. Palette directory of inkstitch
        """
        path = [os.path.join(guess_inkscape_config_path(), 'palettes')]

        if getattr(sys, 'frozen', None) is not None:
            inkstitch_path = os.path.join(sys._MEIPASS, "..")
        else:
            inkstitch_path = dirname(dirname(dirname(realpath(__file__))))

        path.append(os.path.join(inkstitch_path, 'palettes'))

        return path

    def load_palettes(self, paths):
        palettes = []
        for path in paths:
            for palette_file in glob(os.path.join(path, 'InkStitch*.gpl')):
                palette_basename = os.path.basename(palette_file)
                if palette_basename not in palettes:
                    palette = ThreadPalette(palette_file)
                    if not palette.is_gimp_palette:
                        continue
                    self.palettes.append(ThreadPalette(palette_file))
                    palettes.append(palette_basename)

    def palette_names(self):
        return list(sorted(palette.name for palette in self))

    def __getitem__(self, item):
        return self.palettes[item]

    def __len__(self):
        return len(self.palettes)

    def _num_exact_color_matches(self, palette, threads):
        """Number of colors in stitch plan with an exact match in this palette."""

        return sum(1 for thread in threads if thread in palette)

    def match_and_apply_palette(self, stitch_plan, palette=None):
        if palette is None:
            palette = self.match_palette(stitch_plan)
        else:
            palette = self.get_palette_by_name(palette)

        if palette is not None:
            self.apply_palette(stitch_plan, palette)

        return palette

    def match_palette(self, stitch_plan):
        """Figure out which color palette was used

        Scans the catalog of color palettes and chooses one that seems most
        likely to be the one that the user used.  A palette will only be
        chosen if more than 80% of the thread colors in the stitch plan are
        exact matches for threads in the palette.
        """
        if not self.palettes:
            return None

        threads = [color_block.color for color_block in stitch_plan]
        palettes_and_matches = [(palette, self._num_exact_color_matches(palette, threads))
                                for palette in self]
        palette, matches = max(palettes_and_matches, key=lambda item: item[1])

        if matches < 0.8 * len(stitch_plan):
            # if less than 80% of the colors are an exact match,
            # don't use this palette
            return None
        else:
            return palette

    def apply_palette(self, stitch_plan, palette):
        for color_block in stitch_plan:
            nearest = palette.nearest_color(color_block.color)

            color_block.color.name = nearest.name
            color_block.color.number = nearest.number
            color_block.color.manufacturer = nearest.manufacturer

    def get_palette_by_name(self, name):
        for palette in self:
            if palette.name == name:
                return palette

_catalog = None

def ThreadCatalog():
    """Singleton _ThreadCatalog factory"""

    global _catalog
    if _catalog is None:
        _catalog = _ThreadCatalog()

    return _catalog


# coding: utf-8
from django.conf.urls import patterns, url

urlpatterns = patterns(
    'estofadora.statement.views',
    url(r'^caixa/$', 'cash', name='cash'),
    url(r'^caixa-mensal/$', 'cash_month', name='cash_month'),
    url(r'^caixa-anual/$', 'cash_annual', name='cash_annual'),
    url(r'^caixa/(?P<pk>\d+)/remover/', 'delete', name='delete'),
    url(r'^caixa/(?P<pk>\d+)/editar/', 'edit', name='edit'),
    url(r'^$', 'home', name='home'),
)


"""2nd migration

Revision ID: b7e44c457eec
Revises:
Create Date: 2022-09-27 17:17:47.785161

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'b7e44c457eec'
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('authors',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('name', sa.String(length=32), nullable=True),
    sa.Column('book_id', sa.Integer(), nullable=True),
    sa.ForeignKeyConstraint(['book_id'], ['books.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_index(op.f('ix_authors_name'), 'authors', ['name'], unique=True)
    op.create_table('association',
    sa.Column('book_id', sa.Integer(), nullable=False),
    sa.Column('author_id', sa.Integer(), nullable=False),
    sa.ForeignKeyConstraint(['author_id'], ['authors.id'], ),
    sa.ForeignKeyConstraint(['book_id'], ['books.id'], ),
    sa.PrimaryKeyConstraint('book_id', 'author_id')
    )
    op.create_index(op.f('ix_books_title'), 'books', ['title'], unique=False)
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_index(op.f('ix_books_title'), table_name='books')
    op.drop_table('association')
    op.drop_index(op.f('ix_authors_name'), table_name='authors')
    op.drop_table('authors')
    # ### end Alembic commands ###


import RPi.GPIO as GPIO
import time
import socketio
from socketio.exceptions import ConnectionError

def main():
    sio = None

    GPIO.setmode(GPIO.BOARD)

    # Setup PowerLED + input listener on PIN 3
    input_channel = 3
    GPIO.setup(input_channel, GPIO.IN)

    # Setup PowerSwitch RELAY control on PIN 12
    output_channel = 12
    GPIO.setup(output_channel, GPIO.OUT, initial=GPIO.LOW)

    count = 0

    while True:
        try:
            # Read voltage value from PIN 3
            input_voltage = GPIO.input(input_channel)

            if input_voltage == GPIO.LOW:
                # Detect power off, count once
                count += 1
            else:
                # Reset count to zero if value is GPIO.HIGH
                count = 0

            time.sleep(0.1)

            # Press Power Switch for 0.3 seconds when system is powered off for 10 seconds.
            if count > 100:
                # Simulate cold boot
                GPIO.output(output_channel, GPIO.HIGH)
                time.sleep(0.3)
                GPIO.output(output_channel, GPIO.LOW)

                # Update success operation count to Flask WebServer
                if sio:
                    sio.emit("count_plus_one")
                else:
                    try:
                        sio = socketio.Client()
                        sio.connect("http://localhost/")
                        sio.emit("count_plus_one")
                    except ConnectionError:
                        sio = None
                count = 0
        except KeyboardInterrupt:
            GPIO.cleanup()
            exit(0)

if __name__ == "__main__":
    main()


from django.urls import path

from . import views

urlpatterns = [
    path("", views.index, name="index"),
    path("following", views.following, name="following"),
    path("user/<str:username>", views.user, name="user"),

    path("login", views.login_view, name="login"),
    path("logout", views.logout_view, name="logout"),
    path("register", views.register, name="register"),

    path("create", views.create_post, name="create"),
    path("edit", views.edit, name="edit"),
    path("like/<int:id>", views.likes, name="like"),
    path("follow/<str:username>", views.follow, name="follow"),

    path("posts/page<int:pagenumber>", views.posts, name="posts"),
    path("followposts/page<int:pagenumber>", views.followposts, name="followposts"),
    path("userposts/<str:username>/page<int:pagenumber>", views.userposts, name="userposts"),

    path("getlikes", views.get_likes, name="getlikes"),
    path("getuser", views.getuser, name="getuser"),
    path("getpages", views.getpages, name="getpages"),

    path("flock", views.flock, name="flock")
]


# -*- coding: utf-8 -*-

import uuid
import re

from django.utils.translation import gettext_lazy as _
from django.db import models
from django.utils import timezone
from django.contrib.auth.models import (AbstractUser,
                                        Permission,
                                        PermissionsMixin)
from django.core import validators
from userauth.managers import (ElectorUserManager,
                               ElectorDynamicUserManager)

class ElectoralUser(AbstractUser,
                    PermissionsMixin):
    """
    User model for electoral application
    Required Fields :: (email, full_name)
    """
    ROLES = (
        ('AD', 'Admin'),
        ('WK', 'Worker'),
        ('MN', 'Manager'),
        ('ST', 'Support'),
        ('IN', 'Internal')
    )
    uuid = models.UUIDField(default=uuid.uuid4,
                            editable=False)
    username = models.CharField(_("user name"),
                                max_length=32,
                                unique=True,
                                null=True,
                                blank=True,
                                help_text='Required. 30 characters or fewer. Letters, numbers and /./-/_characters',
                                validators=[validators.RegexValidator(re.compile(r"^[\w.-]+$"),
                                                                      "Enter a valid username.",
                                                                      "invalid")])
    email = models.EmailField(_("email"),
                              max_length=255,
                              null=False,
                              blank=False,
                              unique=True)
    role = models.CharField(max_length=10,
                            choices=ROLES,
                            default='IN')
    full_name = models.CharField(_("full name"),
                                 max_length=256,
                                 blank=False)
    is_superuser = models.BooleanField(default=False,
                                       verbose_name='superuser status',
                                       help_text="Designates whether the user can have access"
                                                 "to all permissions on every model."
                                       )
    is_active = models.BooleanField(_('active status'),
                                    default=True,
                                    help_text="Designates whether this user should be treated as active."
                                              "Unselect this instead of deleting accounts.")
    is_staff = models.BooleanField(_('staff status'),
                                   default=False,
                                   help_text="Designates whether the user can log into this admin site.")
    full_name = models.CharField(_("full name"),
                                 max_length=256,
                                 blank=True)
    bio = models.TextField(_("biography"),
                           null=False,
                           blank=True,
                           default="")
    photo = models.ImageField(upload_to='profile-pictures',
                              blank=True,
                              null=True)
    date_joined = models.DateTimeField(verbose_name='date joined',
                                       default=timezone.now)
    added_by = models.ForeignKey('ElectoralUser',
                                 related_name='added_by_elector_users',
                                 on_delete=models.SET_NULL,
                                 null=True,
                                 blank=True)

    objects = ElectorUserManager()

    EMAIL_FIELD = 'email'
    USERNAME_FIELD = 'email'
    REQUIRED_FIELDS = ['full_name']

    class Meta:
        verbose_name = 'Electoral User'
        verbose_name_plural = 'Electoral Users'
        ordering = ['full_name', '-date_joined']

class AdminUser(ElectoralUser):
    objects = ElectorDynamicUserManager('AD')

    class Meta:
        proxy = True
        verbose_name = "Admin User"
        verbose_name_plural = "Admin Users"

class ManagerUser(ElectoralUser):
    objects = ElectorDynamicUserManager('MN')

    class Meta:
        proxy = True
        verbose_name = "Manager User"
        verbose_name_plural = "Manager Users"

class WorkerUser(ElectoralUser):
    objects = ElectorDynamicUserManager('WK')

    class Meta:
        proxy = True
        verbose_name = "Worker User"
        verbose_name_plural = "Worker Users"


from rich import print
from rich.prompt import Prompt
from .base import Environment
from adala.skills import SkillSet
from adala.utils.internal_data import InternalDataFrame
from adala.utils.logs import print_series
from typing import Optional
from collections import defaultdict
from adala.environments.base import EnvironmentFeedback

class ConsoleEnvironment(Environment):
    def get_feedback(
        self,
        skills: SkillSet,
        predictions: InternalDataFrame,
        num_feedbacks: Optional[int] = None,
    ):
        if num_feedbacks is not None:
            predictions = predictions.sample(n=num_feedbacks)

        feedback = defaultdict(list)
        match = defaultdict(list)
        for skill_output, skill_name in skills.get_skill_outputs():
            for _, prediction in predictions.iterrows():
                print_series(prediction)
                print(
                    f'Prediction for "{skill_name}": {skill_output} = {prediction[skill_output]}'
                )
                fb = Prompt.ask(
                    "Your feedback: (Correct/Incorrect/Provide your answer)",
                    default="Correct",
                )
                if fb == "Correct":
                    match[skill_output].append(True)
                else:
                    match[skill_output].append(False)
                feedback[skill_output].append(fb)

        return EnvironmentFeedback(
            match=InternalDataFrame(match, index=predictions.index),
            feedback=InternalDataFrame(feedback, index=predictions.index),
        )


import youtube_dl
import subprocess
import os.path
ydl_opts = {
    'format': 'bestaudio/best',
    'postprocessors': [{
        'key': 'FFmpegExtractAudio',
        'preferredcodec': 'vorbis',
        'preferredquality': '192',
    }],
}

songs = [
    ('http://www.youtube.com/watch?v=0a6j3rv1PFk', '我會守在這裡'),
    ('https://www.youtube.com/watch?v=BiiwclhI5Y8', '平凡的一天')
]

for url, out_name in songs:
    ydl_opts['outtmpl'] = '{}/{}.%(ext)s'.format(out_name, out_name)
    if not os.path.isfile('{}/{}.ogg'.format(out_name,out_name)):
        with youtube_dl.YoutubeDL(ydl_opts) as ydl:
            ydl.download([url])
    else:
        print("{} already exists, skipping".format(out_name))

subprocess.call(['ffmpeg',  '-ss', '22', '-i', '平凡的一天/平凡的一天.ogg', '-acodec', 'copy', '平凡的一天/平凡的一天_cut.ogg'])


# -*- coding: utf-8 -*-
# @Time    : 2023/7/3 8:43 下午
# @Author  : 顾安
# @File    : 5.事件循环的创建.py
# @Software: PyCharm

"""
python3.7以上版本中不需要我们自己手动创建事件循环
    在代码底层自动帮助我们创建

使用python3.6的语法创建一个事件循环
"""

import asyncio

async def work_1():
    for _ in range(5):
        print('协程任务 - 1')
        await asyncio.sleep(1)

async def work_2():
    for _ in range(5):
        print('协程任务 - 2')
        await asyncio.sleep(1)

loop = asyncio.get_event_loop()  # 创建一个事件循环

# 当前列表中存储的是协程对象
coroutines = [
    work_1(),
    work_2()
]

done, pending = loop.run_until_complete(asyncio.wait(coroutines))

for item in done:
    print(type(item))

"""
1.事件循环如何创建 - get_event_loop
2.如何去运行事件循环 - run_until_complete
"""

5.事件循环的创建.py


# encoding: utf-8

from __future__ import absolute_import, unicode_literals

import pathvalidate
import typepy
from six.moves import zip

from ._csv import CsvTableWriter

class LtsvTableWriter(CsvTableWriter):
    """
    A table writer class for
    `Labeled Tab-separated Values (LTSV) <http://ltsv.org/>`__ format.

        :Example:
            :ref:`example-ltsv-table-writer`
    """

    FORMAT_NAME = "ltsv"

    @property
    def format_name(self):
        return self.FORMAT_NAME

    @property
    def support_split_write(self):
        return True

    def __init__(self):
        super(LtsvTableWriter, self).__init__()

        self.is_write_header = False

        self._is_require_header = True

    def write_table(self):
        """
        |write_table| with
        `Labeled Tab-separated Values (LTSV) <http://ltsv.org/>`__ format.
        Invalid characters in labels/data are removed.

        :raises pytablewriter.EmptyHeaderError: If the |headers| is empty.
        :Example:
            :ref:`example-ltsv-table-writer`
        """

        with self._logger:
            self._verify_property()
            self._preprocess()

            for values in self._table_value_matrix:
                ltsv_item_list = [
                    "{:s}:{}".format(pathvalidate.sanitize_ltsv_label(header_name), value)
                    for header_name, value in zip(self.headers, values)
                    if typepy.is_not_null_string(value)
                ]

                if typepy.is_empty_sequence(ltsv_item_list):
                    continue

                self._write_line("\t".join(ltsv_item_list))


#!/usr/bin/env python

import hashlib
import json
import logging
from pathlib import Path
import sys

import pandas as pd

CLADEMAP = {
    'Arabidopsis thaliana': {'Angiosperm', 'Dicot'},
    'Asparagus officinalis': {'Angiosperm', 'Monocot'},
    'Brachypodium distachyon': {'Angiosperm', 'Monocot', 'Grass'},
    'Eutrema salsugineum': {'Angiosperm', 'Dicot'},
    'Glycine max': {'Angiosperm', 'Dicot'},
    'Hordeum vulgare': {'Angiosperm', 'Monocot', 'Grass'},
    'Oryza sativa': {'Angiosperm', 'Monocot', 'Grass'},
    'Phaseolus vulgaris': {'Angiosperm', 'Dicot'},
    'Populus trichocarpa': {'Angiosperm', 'Dicot'},
    'Setaria viridis': {'Angiosperm', 'Monocot', 'Grass'},
    'Sorghum bicolor': {'Angiosperm', 'Monocot', 'Grass'},
    'Spirodela polyrhiza': {'Angiosperm', 'Monocot'},
    'Vitis vinifera': {'Angiosperm', 'Dicot'},
    'Zea mays': {'Angiosperm', 'Monocot', 'Grass'}
}

logging.basicConfig(level = logging.DEBUG)

if len(sys.argv) != 2:
    logging.error("Usage: %s config.json" % sys.argv[0])
    sys.exit(1)

config_path = Path(sys.argv[1])
with open(config_path) as config_file:
    config = json.load(config_file)

cmdline = 'mkdir -p "{out_dir}" && TF_CPP_MIN_LOG_LEVEL=1 python train.py -v -v "{config_path}" "{out_dir}" "{test_species}" "{train_species}" 2> "{out_dir}/train.err" > "{out_dir}/train.out"'

split_hash = hashlib.sha256((json.dumps(config['paths'], indent = 2) + "\n" + json.dumps(config['preprocessing'], indent = 2) + "\n" + json.dumps(config['splitting'], indent = 2) + "\n").encode('utf-8')).hexdigest()
config_hash = hashlib.sha256((json.dumps(config, indent = 2) + "\n").encode('utf-8')).hexdigest()
samples = config['training']['samples']

# subset species to within clade
clade = config['training']['clade']
clade_set = set(filter(lambda s: clade in CLADEMAP[s], CLADEMAP.keys()))

split_data_path = f"tmp/data/split/{split_hash}.tsv"
available_species = set(pd.read_table(split_data_path, header = 0, usecols = ['species'])['species'].unique())

available_clade_species = available_species & clade_set
if not available_clade_species:
    logging.error("No species in the data (%s) are in the clade '%s'" % (split_data_path, clade))
    sys.exit(1)

if config['training']['strategy'] == 'drop-one':
    # train on all other species in given clade
    for test_species in available_clade_species:
        for sample in range(samples):
            out_dir = Path(f"tmp/results/{config_path.stem}/{test_species.replace(' ', '_')}/{sample}")
            if out_dir.exists():
                logging.warning("%s exists, skipping that job" % out_dir)
            else:
                sys.stdout.write(cmdline.format(
                    config_path = config_path,
                    test_species = test_species,
                    train_species = ','.join(available_clade_species - {test_species}),
                    out_dir = out_dir
                ) + "\n")
elif config['training']['strategy'] == 'within':
    # train within species in given clade
    for test_species in available_clade_species:
        for sample in range(samples):
            out_dir = Path(f"tmp/results/{config_path.stem}/{test_species.replace(' ', '_')}/{sample}")
            if (out_dir / 'metrics.json').exists():
                logging.warning("%s/metrics.json exists, skipping that job" % out_dir)
            else:
                sys.stdout.write(cmdline.format(
                    config_path = config_path,
                    test_species = test_species,
                    train_species = test_species,
                    out_dir = out_dir
                ) + "\n")
else:
    logging.error("Unknown training strategy '%s'" % config['training']['strategy'])
    sys.exit(1)


import os
import cv2

""" Unorders the data.
    This supposes that the original images and their labels are in separate folders
    containing sub-folders 200_2, 200_4 etc..
    The outputs are folders containing the original images and the labels named like the following:
    image_x.tif and label_x.jpg were x is an integer. An original image and its label have the same number. """

def main():

    root_path = os.path.join(os.pardir, "ternausnet/data/labels")

    save_path_labels = os.path.join(os.pardir, "ternausnet/data/labels" + "_unordered/")
    save_path_originals = os.path.join(os.pardir, "ternausnet/data/originals" + "_unordered/")

    if not os.path.exists(save_path_labels):
        os.makedirs(save_path_labels)

    if not os.path.exists(save_path_originals):
        os.makedirs(save_path_originals)

    img_num = 1

    for root, dirs, files in os.walk(root_path, topdown=True):
        for file in files:
            if file.endswith(".jpg"):

                # label
                image = cv2.imread(os.path.join(root, file), cv2.IMREAD_UNCHANGED)
                cv2.imwrite(os.path.join(save_path_labels,  "label_" + str(img_num) + ".jpg"), image)

                num = file.split("_")[-1][:-4]

                #original
                image = cv2.imread(os.path.join(root.replace("labels", "originals"), str(num)+".tif"), cv2.IMREAD_UNCHANGED)
                cv2.imwrite(os.path.join(save_path_originals,  "image_" + str(img_num) + ".tif"), image)
                img_num += 1

if __name__ == '__main__':
    main()


import pandas as pd
import numpy as np
import os
import torch
import os
from tqdm import tqdm
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, classification_report

from torch import nn, optim
import torch.nn.functional as F
from torch.utils.data import DataLoader, TensorDataset

class Network(nn.Module):
    def __init__(self):
        super().__init__()

        # Inputs to hidden layer linear transformation
        self.l1 = nn.Linear(40, 60)
        self.l2 = nn.Linear(60, 60)
        self.l3 = nn.Linear(60, 40)
        self.l4 = nn.Linear(40, 10)

        # Output layer, 10 units - one for each digit
        self.output = nn.Linear(10, 1)

        # Define sigmoid activation and softmax output
        self.softmax = nn.Softmax(dim=1)

    def forward(self, x):
        # Pass the input tensor through each of our operations
        x = self.l1(x)
        x = F.relu(x)
        #         x = self.l2(x)
        #         x = F.relu(x)
        x = self.l3(x)
        x = F.relu(x)
        x = self.l4(x)
        x = F.relu(x)

        return self.output(x)

def train_model(model, epochs, loss_function, lr, train_loader, test_loader):
    loss_value = []
    optimizer = optim.Adam(params=model.parameters(), lr=lr)

    for i in range(epochs):
        for (x, y) in train_loader:
            x = x.to(device)
            # initialize the gradient of model parameters
            optimizer.zero_grad()
            # calculate the loss
            y_pred = model(x)

            loss = loss_function(y_pred, y.to(device))

            loss.backward()

            optimizer.step()
        if (i % 1) == 0:
            print('epoch: {},'.format(i) + 'loss: {}'.format(loss.detach()))

if __name__ == "__main__":
    print(os.getcwd())
    df = pd.read_csv('data.csv', sep=',', encoding='utf8', engine="python", index_col=0)
    df = df.dropna()
    df.head(5)

    fields = df.columns
    fields = [_f for _f in fields if _f not in ["season", "player", "contract_signing", "league"]]

    X = df[fields]
    y = df[['wage_eur']]
    X.dtypes
    X = torch.tensor(np.array(X), dtype=torch.float)
    y = torch.tensor(np.array(y).reshape(-1, 1), dtype=torch.float)

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
    # X_train = torch.from_numpy(X_train.to_numpy()).float()
    # y_train = torch.squeeze(torch.from_numpy(y_train.to_numpy()).float())

    # X_test = torch.from_numpy(X_test.to_numpy()).float()
    # y_test = torch.squeeze(torch.from_numpy(y_test.to_numpy()).float())

    print(X_train.shape, y_train.shape)
    print(X_test.shape, y_test.shape)
    train_dataset = TensorDataset(X_train, y_train)
    test_dataset = TensorDataset(X_test, y_test)

    train_loader = DataLoader(train_dataset, batch_size=64)
    test_loader = DataLoader(test_dataset, batch_size=32)

    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

    net = Network().to(device)
    loss_function = nn.MSELoss()

    train_model(net, 100, loss_function, 1e-5, train_loader, test_loader)


import re

from bs4 import BeautifulSoup
from decimal import Decimal

from requests import TooManyRedirects

from storescraper.product import Product
from storescraper.store_with_url_extensions import StoreWithUrlExtensions
from storescraper.utils import session_with_proxy, html_to_markdown, \
    remove_words
from storescraper.categories import NOTEBOOK, ALL_IN_ONE, TABLET, \
    EXTERNAL_STORAGE_DRIVE, SOLID_STATE_DRIVE, MEMORY_CARD, \
    USB_FLASH_DRIVE, PROCESSOR, COMPUTER_CASE, POWER_SUPPLY, MOTHERBOARD, \
    RAM, VIDEO_CARD, MOUSE, PRINTER, HEADPHONES, STEREO_SYSTEM, UPS, MONITOR, \
    KEYBOARD_MOUSE_COMBO, KEYBOARD, GAMING_CHAIR, WEARABLE, \
    STORAGE_DRIVE

class Cintegral(StoreWithUrlExtensions):
    url_extensions = [
        ['109', NOTEBOOK],
        ['110', ALL_IN_ONE],
        ['176', TABLET],
        ['157', PRINTER],
        ['156', PRINTER],
        ['129', PRINTER],
        ['152', PRINTER],
        ['181', STORAGE_DRIVE],
        ['182', EXTERNAL_STORAGE_DRIVE],
        ['183', SOLID_STATE_DRIVE],
        ['184', MEMORY_CARD],
        ['185', USB_FLASH_DRIVE],
        ['116', POWER_SUPPLY],
        ['115', COMPUTER_CASE],
        ['118', RAM],
        ['117', MOTHERBOARD],
        ['114', PROCESSOR],
        ['119', VIDEO_CARD],
        ['96', MONITOR],
        ['122', KEYBOARD_MOUSE_COMBO],
        ['123', MOUSE],
        ['121', KEYBOARD],
        ['161', NOTEBOOK],
        ['163', WEARABLE],
        ['162', TABLET],
        ['164', KEYBOARD],
        ['139', HEADPHONES],
        ['140', STEREO_SYSTEM],
        ['153', GAMING_CHAIR],
        ['86', WEARABLE],
        ['168', VIDEO_CARD],
        ['160', MONITOR],
        ['159', NOTEBOOK],
        ['172', SOLID_STATE_DRIVE],
        ['171', POWER_SUPPLY],
        ['167', MOUSE],
        ['170', MOTHERBOARD],
        ['169', PROCESSOR],
        ['166', GAMING_CHAIR],
        ['178', RAM],
        ['106', NOTEBOOK],
        ['141', UPS],
    ]

    @classmethod
    def discover_urls_for_url_extension(cls, url_extension, extra_args=None):
        product_urls = []
        session = session_with_proxy(extra_args)
        session.headers['Content-Type'] = 'application/x-www-form-urlencoded'
        page = 1

        while True:
            if page >= 20:
                raise Exception('Page overflow: ' + url_extension)

            payload = ('action=jet_smart_filters&provider=jet-engine/default&'
                       'query[_tax_query_product_cat][]={}&'
                       'settings[lisitng_id]=645&referrer[uri]=/'
                       '?jsf=jet-engine&pagenum={}').format(
                url_extension, page)
            print(payload)

            res = session.post('https://cintegral.cl/wp-admin/admin-ajax.php',
                               payload)
            json_data = res.json()
            soup = BeautifulSoup(json_data['content'], 'html.parser')
            product_tags = soup.findAll('div', 'jet-listing-grid__item')

            if not product_tags:
                break

            for product_tag in product_tags:
                product_url = product_tag.find(
                    'a', 'jet-listing-dynamic-image__link')['href']
                product_urls.append(product_url)

            page += 1

        return product_urls

    @classmethod
    def products_for_url(cls, url, category=None, extra_args=None):
        print(url)
        session = session_with_proxy(extra_args)
        try:
            res = session.get(url)
        except TooManyRedirects:
            return []
        soup = BeautifulSoup(res.text, 'html.parser')

        name = soup.find('h1', 'product_title').text.strip()
        new_key = soup.find('link', {'rel': 'shortlink'})['href'].split(
            '?p=')[-1]
        old_key = re.match(r'https://cintegral.cl/producto/(\d+)',
                           url).groups()[0]

        stock_label_tag = soup.find('span', text='Stock Web:')
        stock_tag = stock_label_tag.next.next
        stock_match = re.search(r'(\d+)', stock_tag.text.strip())
        if not stock_match:
            return []
        stock = int(stock_match.groups()[0])

        prices_tag = soup.find('div', 'elementor-element-9386c30')

        normal_price_label_tag = prices_tag.find(
            'div', text='Pago con Tarjeta')
        if normal_price_label_tag:
            normal_price = Decimal(
                remove_words(normal_price_label_tag.next.next.text))

            offer_price_label_tag = prices_tag.find(
                'div', text='Pago transferencia')
            offer_price = Decimal(remove_words(
                offer_price_label_tag.next.next.text))
        else:
            price_label_tag = prices_tag.find('div', text='Todo Medio de Pago')
            price = Decimal(
                remove_words(price_label_tag.next.next.text))
            normal_price = offer_price = price

        mpn_label_tag = prices_tag.find('span', text='Part Number:')
        if mpn_label_tag:
            part_number = mpn_label_tag.next.next.strip()
        else:
            part_number = None

        description = html_to_markdown(str(
            soup.find('div', 'woocommerce-Tabs-panel--description')))

        picture_urls = [soup.find('meta', {'property': 'og:image'})['content']]

        if 'REACONDICIONADO' in name.upper():
            condition = 'https://schema.org/RefurbishedCondition'
        else:
            condition = 'https://schema.org/NewCondition'

        p = Product(
            name,
            cls.__name__,
            category,
            url,
            url,
            new_key,
            stock,
            normal_price,
            offer_price,
            'CLP',
            sku=old_key,
            part_number=part_number,
            description=description,
            picture_urls=picture_urls,
            condition=condition
        )

        return [p]


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='FishQuery',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('location', models.CharField(max_length=200)),
                ('location_id', models.IntegerField()),
                ('species', models.CharField(max_length=200)),
                ('species_id', models.IntegerField()),
                ('year', models.IntegerField()),
            ],
            options={
                'verbose_name_plural': 'Fish Queries',
            },
        ),
    ]


"""Tests for mir.frelia.enja module."""

import io

import pytest

import mir.frelia.enja as enja

def test_load():
    """Test parsing a simple enja document from a file."""
    file = io.StringIO('foo: bar\n---\n
Hello world!
')
    doc = enja.load(file)
    assert doc.header == {'foo': 'bar'}
    assert doc.body == '
Hello world!
'

def test_load_empty_file():
    """Test parsing an empty enja document from a file."""
    file = io.StringIO('')
    doc = enja.load(file)
    assert doc.header == {}
    assert doc.body == ''

def test_dump():
    """Test dumping an Enja document to a file."""
    document = enja.Document('girl meets girl')
    document.header['sophie'] = 'prachta'
    file = io.StringIO()
    enja.dump(document, file)
    assert file.getvalue() == 'sophie: prachta\n---\ngirl meets girl'

def test_document_equal():
    """Test document equality."""
    doc1 = enja.Document('girl meets girl')
    doc1.header['sophie'] = 'prachta'
    doc2 = enja.Document('girl meets girl')
    doc2.header['sophie'] = 'prachta'
    assert doc1 == doc2

def test_document_unequal_body():
    """Test document with unequal body."""
    doc1 = enja.Document('girl meets girl')
    doc1.header['sophie'] = 'prachta'
    doc2 = enja.Document('')
    doc2.header['sophie'] = 'prachta'
    assert doc1 != doc2

def test_document_unequal_header():
    """Test document with unequal header."""
    doc1 = enja.Document('girl meets girl')
    doc1.header['sophie'] = 'prachta'
    doc2 = enja.Document('girl meets girl')
    assert doc1 != doc2

def test_document_unequal_type():
    """Test document against unequal type."""
    doc = enja.Document('girl meets girl')
    doc.header['sophie'] = 'prachta'
    assert doc != 'girl meets girl'

def test_document_repr():
    """Test document repr()."""
    doc = enja.Document('girl meets girl')
    doc.header['sophie'] = 'prachta'
    assert repr(doc) == "<Document with header={'sophie': 'prachta'}, body='girl meets girl'>"


import pandas as pd
import numpy as np
import logging
from sklearn.preprocessing import StandardScaler
import os
import time
import tensorflow as tf
import pickle

import config

from src.dataset import get_data, get_features_targets, merge_data, split_data, extract_most_important_features, get_lgbm_features, \
    create_flatten_features, create_lstm_features
from src.indicators import get_indicators, get_price_patterns, get_additional_indicators
from src.logger import setup_logger
from src.models import BiLSTMModel, LightGBMModel, evaluate

log = setup_logger(stderr_level=logging.INFO)
scaler = StandardScaler()

if __name__ == '__main__':

    log.info("Starting test data reading.")
    df = get_data(config.TEST_DATA_PATH, drop_col=config.TEST_DROP_COLS)
    log.info("Getting indicator for data.")
    df_indicators = get_indicators(df, intervals=config.INTERVALS)
    log.info("Getting price pattern for data.")
    df_price_pattern = get_price_patterns(df)
    log.info("Getting additional indicators.")
    df_add_indicators = get_additional_indicators(df)
    log.info("Merging all data into one.")
    data = merge_data(df, df_indicators, df_price_pattern, df_add_indicators, test=True)
    print(data.head())

    log.info("Loading features names from pickle file.")
    # with open(config.feature_save_path, 'rb') as f:
    #     features_names = pickle.load(f)

    features_names = config.test_fe_names
    log.info("Getting features and targets for training data.")
    features, _ = get_features_targets(data, None, features_names, date_col='Date')
    log.info(f"Shape of test features: {features.shape}")

    features = features.values
    features, _ = create_flatten_features(features, None, config.n_context, features_names)
    log.info(f"Shape of test features: {features.shape}.")

    log.info("Initializing LightGBM model.")
    model = LightGBMModel(config.model_parameters, config.fit_parameters, config, inference=True)

    predictions = model.predict(features)
    log.info(f"Shape of the predictions {predictions.shape}")

    df['Predictions'] = list(range(config.n_context)) + predictions.tolist()
    df['Class'] = list(range(config.n_context)) + np.round(predictions).tolist()

    print(df.head())
    df = df.sort_values(by='Date', ascending=False).reset_index(drop=True)
    os.makedirs(f'./data/{config.TARGET}', exist_ok=True)
    df.to_csv(f"./data/{config.TARGET}/btc_predictions_{config.TARGET}_{str(time.time())}.csv", index=False)