import torch
from torch.autograd import Variable
import torch.functional as F
from dataLoader import BatchLoader
import argparse
import torch.optim as optim
import torchvision.utils as vutils
from torch.utils.data import DataLoader
import torch.nn as nn
import os
import numpy as np
import utils
import scipy.io as io
import model
import utils

parser = argparse.ArgumentParser()
parser.add_argument('--model', default='basic', type=str)
parser.add_argument('--imageRoot', default='/datasets/cs252csp19-public/VOCdevkit/VOC2012/JPEGImages', help='path to input images' )
parser.add_argument('--labelRoot', default='/datasets/cs252csp19-public/VOCdevkit/VOC2012/SegmentationClass', help='path to input images' )
parser.add_argument('--fileList', default='/datasets/cs252csp19-public/VOCdevkit/VOC2012/ImageSets/Segmentation/train.txt', help='path to input images' )
parser.add_argument('--experiment', default='checkpoint', help='the path to store sampled images and models' )
parser.add_argument('--epoches', type=int, default=24, help='the number of epochs being trained')
parser.add_argument('--batchSize', type=int, default=32, help='the size of a batch')
parser.add_argument('--numClasses', type=int, default=21, help='the number of classes')
parser.add_argument('--isDilation', action='store_true', help='whether to use dialated model or not' )
parser.add_argument('--isSpp', action='store_true', help='whether to do spatial pyramid or not' )
parser.add_argument('--noCuda', action='store_true', help='do not use cuda for training' )
parser.add_argument('--num_workers', default=0, help='number of workers used in PyTorch')
parser.add_argument('--gpuId', type=int, default=0, help='gpu id used for training the network' )
parser.add_argument('--colormap', default='colormap.mat', help='colormap for visualization')
parser.add_argument('--imHeight', default=300, type=int)
parser.add_argument('--imWidth', default=300, type=int)
parser.add_argument('--iterationDecreaseLR', default=800, type=int)
parser.add_argument('--iterationEnd', default=3000)

# The detail network setting
opt = parser.parse_args()
print(opt)

# Not necessary for the training process
colormap = io.loadmat(opt.colormap )['cmap']

if opt.isSpp == True :
    opt.isDilation = False

if opt.isDilation:
    opt.experiment += '_dilation'
if opt.isSpp:
    opt.experiment += '_spp'

os.mkdir(opt.experiment)

# Initialize network
if opt.isDilation:
    encoder = model.encoderDilation()
    decoder = model.decoderDilation()
elif opt.isSpp:
    encoder = model.encoderSPP()
    decoder = model.decoderSPP()
else:
    encoder = model.encoder()
    decoder = model.decoder()

model.loadPretrainedWeight(encoder, False)

encoder = encoder.cuda(opt.gpuId)
decoder = decoder.cuda(opt.gpuId)

# Initialize variables for training process
imBatch = Variable(torch.FloatTensor(opt.batchSize, 3, opt.imHeight, opt.imWidth) )
labelBatch = Variable(torch.FloatTensor(opt.batchSize, opt.numClasses, opt.imHeight, opt.imWidth) )
maskBatch = Variable(torch.FloatTensor(opt.batchSize, 1, opt.imHeight, opt.imWidth) )
labelIndexBatch = Variable(torch.LongTensor(opt.batchSize, 1, opt.imHeight, opt.imWidth) )

# Put variables on GPU
if not opt.noCuda:
    imBatch = imBatch.cuda(opt.gpuId)
    labelBatch = labelBatch.cuda(opt.gpuId)
    labelIndexBatch = labelIndexBatch.cuda(opt.gpuId)
    maskBatch = maskBatch.cuda(opt.gpuId)
    encoder = encoder.cuda(opt.gpuId)
    decoder = decoder.cuda(opt.gpuId)

# Load data
segDataset = BatchLoader(imageRoot = opt.imageRoot,
                    labelRoot = opt.labelRoot,
                    fileList = opt.fileList,
                    imWidth = opt.imWidth, imHeight = opt.imHeight)
segLoader = DataLoader(segDataset, batch_size=opt.batchSize, num_workers=opt.num_workers, shuffle=True)

# function to calculate mIoU accuracy during training
def calculate_mIoU_accuracy(pred, labelBatch, labelIndexBatch, maskBatch, numClasses):
    '''
    Calculage mIoU during training, given the temporary results during during

    code mainly borrowd from test.py
    '''
    confcounts = np.zeros( (numClasses, numClasses), dtype=np.int64 )
    accuracy = np.zeros(numClasses, dtype=np.float32 )
    # Compute mean IOU
    hist = utils.computeAccuracy(pred, labelIndexBatch, maskBatch )
    confcounts += hist

    for n in range(0, numClasses):
        rowSum = np.sum(confcounts[n, :] )
        colSum = np.sum(confcounts[:, n] )
        interSum = confcounts[n, n]
        accuracy[n] = float(100.0 * interSum) / max(float(rowSum + colSum - interSum ), 1e-5)

    meanAccuracy = np.mean(accuracy )

    return meanAccuracy

# Set up optimizer, we use deffirent learning rate for encoder (small) and decoder (large)
lr_encoder = 1e-4
lr_decoder = 1e-2

optimizer = optim.Adam([
                {'params': encoder.parameters(), 'lr': lr_encoder},
                {'params': decoder.parameters(), 'lr': lr_decoder}
                ]) # encoder has lr=1e-3 and decoder has 1e-1

iterationDecreaseLR = opt.iterationDecreaseLR

lossArr = []
accuracyArr = []
iteration = 0

#lossFunction = torch.mean()

for epoch in range(opt.epoches):
    print('Starting epoch {}/{}.'.format(epoch+1, opt.epoches))
    trainingLog = open('{0}/trainingLog_{1}.txt'.format(opt.experiment, epoch), 'w')

    for i, dataBatch in enumerate(segLoader):
        iteration += 1

        # Read data
        image_cpu = dataBatch['im']
        imBatch.data.resize_(image_cpu.size() )
        imBatch.data.copy_(image_cpu )

        label_cpu = dataBatch['label']
        labelBatch.data.resize_(label_cpu.size() )
        labelBatch.data.copy_(label_cpu )

        labelIndex_cpu = dataBatch['labelIndex' ]
        labelIndexBatch.data.resize_(labelIndex_cpu.size() )
        labelIndexBatch.data.copy_(labelIndex_cpu )

        mask_cpu = dataBatch['mask' ]
        maskBatch.data.resize_( mask_cpu.size() )
        maskBatch.data.copy_( mask_cpu )

        # Train network
        optimizer.zero_grad()

        # Feed forward
        x1, x2, x3, x4, x5 = encoder(imBatch)
        pred = decoder(imBatch, x1, x2, x3, x4, x5)

        # Calculate Loss and back propergation
        #loss = lossFunction( pred * labelBatch )
        loss = torch.mean(pred * labelBatch)
        loss.backward()

        # Calculate mIoU accuracy each iteration as a mark to monitor the training process
        accuracy = calculate_mIoU_accuracy(pred, labelBatch, labelIndexBatch, maskBatch, opt.numClasses)

        # update training step
        optimizer.step()

        # Record training progress
        lossArr.append(loss.cpu().data.item())
        accuracyArr.append(accuracy)

        # Calculate mean training loss (pixel-wise cross-entropy) and mean training accuracy (mIoU)
        if iteration >= 200:
            meanLoss = np.mean(np.array(lossArr[-1000:] ) )
            meanAccuracy = np.mean(np.array(accuracyArr[-1000:] ) )
        else:
            meanLoss = np.mean(np.array(lossArr[:] ) )
            meanAccuracy = np.mean(np.array(accuracyArr[:] ) )

        print('Epoch %d iteration %d: Loss %.5f Accumulated Loss %.5f' % (epoch+1, iteration, lossArr[-1], meanLoss ) )
        print('Epoch %d iteration %d: Accura %.5f Accumulated Accura %.5f' % (epoch+1, iteration, accuracyArr[-1], meanAccuracy ) )
        trainingLog.write('Epoch %d iteration %d: Loss %.5f Accumulated Loss %.5f \n' % (epoch+1, iteration, lossArr[-1], meanLoss ) )
        trainingLog.write('Epoch %d iteration %d: Accura %.5f Accumulated Accura %.5f\n' % (epoch+1, iteration, accuracyArr[-1], meanAccuracy ) )
        if iteration == iterationDecreaseLR and lr_encoder > 1e-4:
            print('The learning rate is being decreased at iteration %d' % iteration )
            trainingLog.write('The learning rate is being decreased at iteration %d\n' % iteration )
            for param_group in optimizer.param_groups:
                param_group['lr'] /= 5
                print('New learning rate = %f' % (param_group['lr']))
            iterationDecreaseLR *= 2
            lr_encoder /= 5.
            lr_decoder /= 5.

    print('-' * 80)

    if iteration == opt.iterationEnd:
        np.save('%s/loss.npy' % opt.experiment, np.array(lossArr ) )
        np.save('%s/accuracy.npy' % opt.experiment, np.array(accuracyArr ) )
        torch.save(encoder.state_dict(), '%s/encoderFinal_%d.pth' % (opt.experiment, epoch+1) )
        torch.save(decoder.state_dict(), '%s/decoderFinal_%d.pth' % (opt.experiment, epoch+1) )
        break

    trainingLog.close()

    if iteration >= opt.iterationEnd:
        break

    if (epoch+1) % 2 == 0:
        np.save('%s/loss.npy' % opt.experiment, np.array(lossArr ) )
        np.save('%s/accuracy.npy' % opt.experiment, np.array(accuracyArr ) )
        torch.save(encoder.state_dict(), '%s/encoder_%d.pth' % (opt.experiment, epoch+1) )
        torch.save(decoder.state_dict(), '%s/decoder_%d.pth' % (opt.experiment, epoch+1) )


import csv
import numpy as np
import matplotlib.pyplot as plt
import cv2
from preprocess import image_preprocess

class DataPreparator:
    """
    This class reads a CSV file and prepares a list of the paths of the images and specifies what augmentations to apply later on
    It does NOT read images or apply transformations. It only generates a list with the required transformations
    This allows the usage of generator when reading images

    Attributes
    ----------
    X_train : list of tuples
        A list of tuples - (image path, should we flip this images, brightness adjust factor)
    y_train : list
        The angles for the each image are stored here. Same indexing as X_train
    """
    X_train = []
    y_train = []
    directory = '/'

    def set_prepend_directory(self, directory):
        """
        Changes what is the directory name we should prepend to the path from the CSV file
        Used when parsing CSV files with relative paths.

        Parameters
        ----------
        directory : string
            The path to append in front
        """
        self.directory = directory + '/'

    def _prepend_path(self, filename):
        return self.directory + '/' + filename

    def _append_to_list(self, path, angle, flip = False, brightness_adjust = 1.0):
        """
        Adds one image to the X/y_train list

        Parameters
        ----------
        path : string
            The path to the image
        angle : float
            Steering angle
        flip : boolean
            Should we flip the image when reading the list
        brightness_adjusts: float
            Brightness adjust factor
        """
        self.X_train.append((self._prepend_path(path), flip, brightness_adjust))
        if flip: #Flipped images angle needs correction
            angle = angle * -1
        self.y_train.append(angle)

    def _append_brightness_adjusts(self, path, angle, flip, brightness_adjusts = [1.0]):
        """
        Specifies what the needed brightness adjusts for a specific image are and appends them to X/y_train

        Parameters
        ----------
        path : string
            The path to the image
        angle : float
            Steering angle
        flip : boolean
            Should we flip the image when reading the list
        brightness_adjusts: list
            List of floats specifying what brightness adjusts to add
        """
        for brightness in brightness_adjusts:
            self._append_to_list(path, angle, flip, brightness)

    def _append_adjusted_images(self, row):
        """
        Specifies what the needed transformations for a specific row are and appends them to X/y_train

        Parameters
        ----------
        row : dict
            A row from the CSV file
        """
        angle = float(row[3])
        ANGLE_ADJUST = 0.25 # How much to adjust angle for left/right camera

        # Add the image from the central camera
        if abs(float(row[3])) > 0.01: #Do not flip images with angle close to 0. This helps balance the data
            self._append_brightness_adjusts(row[0], angle, False)
            self._append_brightness_adjusts(row[0], angle, True)
        else:
            self._append_to_list(row[0], angle)

        # Add images from left/right camera
        self._append_brightness_adjusts(row[1], angle + ANGLE_ADJUST, False)
        self._append_brightness_adjusts(row[2], angle - ANGLE_ADJUST, False)
        # Add images from left/right camera, flipped
        self._append_brightness_adjusts(row[1], angle + ANGLE_ADJUST, True)
        self._append_brightness_adjusts(row[2], angle - ANGLE_ADJUST, True)

    def read_csv(self, directory_name, absolute_path = True):
        """
        Reads a CSV file in the Udacity driving log format
        Goes over each row and adds the information to X_train and y_train

        Parameters
        ----------
        directory_name : string
            The directory where drivinig_log.csv and the images are stored
        absolute_path: boolean
            Should we prepend the directory_name to the paths in the CSV file or are they absolute paths
        """
        if not absolute_path:
            self.set_prepend_directory(directory_name)

        with open(directory_name + '/driving_log.csv', newline='') as csvfile:
            logreader = csv.reader(csvfile, delimiter=',', quotechar='|', skipinitialspace=True)
            for row in logreader:
                self._append_adjusted_images(row)

def save_image_data(paths, labels):
    """
    This functions reads lists in the DataPreparator format, opens an image,
    applies transformations and saves them in numpy binary format

    Parameters
    ----------
    paths : string
        The path to append in front

    """
    # Output data format
    X_train = np.empty([len(paths), 16, 32, 1])
    y_train = np.empty([len(paths)])

    # Go over all images
    for index in range(0, len(paths)):
        image = plt.imread(paths[index][0])

        # Apply the needed transformations
        arr = image_preprocess(image, paths[index][2])
        if (paths[index][1]):
            arr = cv2.flip(arr, 1)

        # Store the data for output, reshaping it from 16X32 to 16X32X1
        X_train[index] = arr.reshape((16, 32, 1))
        y_train[index] = labels[index]

    # Save the data in numpy binary format.
    # This speeds up training because we need to do image parsing only once, not before every training
    np.save("x.data", X_train)
    np.save("y.data", y_train)

if __name__ == '__main__':
    # Open the CSV file and parse it
    data_preparator = DataPreparator()
    data_preparator.read_csv(directory_name = 'udacity', absolute_path=False)

    # Save the image data in numpy binary files
    save_image_data(data_preparator.X_train, data_preparator.y_train)


import csv
import json
from PIL import Image
import os

# legge il file csv e restituisce le righe corrispondenti al img_name e al img_text per ricavare i BB
def get_row_csv(csv_name, img_name, img_text):
    annotation_list = []  # una lista perchè nella stessa immagine ci sono parole ripetute
    with open(csv_name) as csv_file:
        csv_reader = csv.reader(csv_file, delimiter=',')
        for riga in csv_reader:
            # print(riga)
            if riga[0][0] == '#':
                riga[0] = riga[0].replace('#', '')  # elimino il cancelletto
            if riga[0] == img_name and riga[9] == img_text:
                annotation_list.append(riga)
        return annotation_list

# ritorna i BB dei caratteri della parola e salva le immagini dei char
def get_crop_img(csv_name, img_name, img_text):
    is_vertical = False  # mi indica se il testo da tagliare è in verticale o orizzontale
    ann_list = get_row_csv(csv_name, img_name, img_text)
    # print(ann_list)
    first_img = ann_list[0]  # se c'è più di un testo uguale, prendo per ora il primo

    im = Image.open(csv_name[:-4] + "/" + first_img[0])  # gli passo il nome dell immagine

    left = int(first_img[1])  # 1.1
    bottom = int(first_img[4])  # 2.2 sarebbe la linea bassa orizzontale
    right = int(first_img[3])  # 2.1
    top = int(first_img[6])  # 3.2 sarebbe la linea alta orizzontale
    if top - bottom > right - left:
        dim_x = top - bottom
        is_vertical = True
    else:
        dim_x = right - left

    os.chdir('../')
    #im1 = im.crop((left, bottom, right, top))  # per vedere il bb della parola
    #im1.show()

    # Opening JSON file
    with open('charLength.json', 'r') as openfile:
        # Reading from json file
        json_object = json.load(openfile)  # è un dizionario

    os.chdir('CSVmarketfiles/')

    # print(json_object)
    somma = 0  # indica la somma dei valori delle lettere che formano il testo
    for letter in img_text:
        somma = somma + json_object[letter]

    charBB = []  # array che andrà a contenere i charBB
    i = 0  # serve per il nome dell immagine
    for letter in img_text:
        crop_dim = json_object[letter] * dim_x / somma
        if is_vertical:
            bottom = top - crop_dim  # vado dall'alto verso il basso
        else:
            right = left + crop_dim
        charBB.append([letter, left, bottom, right, bottom, right, top, left, top])  # scrivo il char e il BB del char

        im1 = im.crop((left, bottom, right, top))
        if is_vertical:
            top = bottom
            im1 = im1.rotate(270)  # ruoto l'immagine
        else:
            left = right

        im1.save('new_img_' + str(i) + letter + img_name)  # salvo l'immagine del carattere
        i += 1

    return charBB

if __name__ == "__main__":
    os.chdir('E-Shop Dataset/CSVmarketfiles/')
    #print(get_crop_img('healthpersonalcare.csv', '0027.jpg', 'ACIDOPHILUS'))
    print(get_crop_img('healthpersonalcare.csv', '0005.jpg', 'HEALTH'))
    print(get_crop_img('grocery.csv', '0017.jpg', 'COINTREAU'))  # bb non rettangolare, risultato non preciso


import random

from django.http import HttpResponse
from django.shortcuts import render

# Create your views here.
def demo_view(request):
    return HttpResponse('Hello world!')
    # return render(request, 'demo.html')

def lotto_view(request):
    game = request.GET.get('game')

    lotto_list = list()
    for index in range(int(game)):

        lotto_number = list()
        for index2 in range(6):
            number = random.randint(1, 45)
            lotto_number.append(number)
        lotto_list.append(lotto_number)

    print(lotto_list)

    if game:
        return render(request, 'lotto.html', {'lotto_list': lotto_list, 'game': game})
    else:
        return render(request, 'lotto.html')


# coding=utf-8
# @author:walter000
# github: https://github.com/Walter000/tencent_competition

"""
 模型训练结束后，将lightgbm的叶子结点onehot处理，再结合原来的特征重新训练得到最终结果
"""

import numpy as np
import pandas as pd
import os
from scipy import sparse
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.preprocessing import StandardScaler
from sklearn.externals import joblib
import xgboost as xgb
import lightgbm as lgb

predict = pd.read_csv('./datasets/test2.csv')
res = predict[['aid', 'uid']]

train_x = sparse.load_npz('./datasets/train.npz')
test_x = sparse.load_npz('./datasets/test.npz')
train_y = pd.read_csv('./datasets/train_label.csv', names=['label'])  # 直接读入label文件，省去了读取train源文件时间
train_y[train_y == -1] = 0

def LGB_train(train_x, train_y):
    print("LGB test")
    clf = lgb.LGBMClassifier(
        boosting_type='gbdt', num_leaves=10, reg_alpha=0.0, reg_lambda=1,
        max_depth=-1, n_estimators=30, objective='binary', scale_pos_weight=3,
        subsample=0.79, colsample_bytree=0.79, subsample_freq=1,
        learning_rate=0.07, min_child_weight=70, random_state=2018, n_jobs=-1
    )
    clf.fit(train_x, train_y, eval_set=[(train_x, train_y)], eval_metric='auc',early_stopping_rounds=100)
    return clf

def LGB_retrain(train_x, train_y, test_x, res):
    clf = lgb.LGBMClassifier(
        boosting_type='gbdt', num_leaves=90, reg_alpha=0.0, reg_lambda=1,
        max_depth=-1, n_estimators=5000, objective='binary', scale_pos_weight=3,
        subsample=0.79, colsample_bytree=0.79, subsample_freq=1,
        learning_rate=0.07, min_child_weight=70, random_state=2018, n_jobs=-1
    )
    clf.fit(train_x, train_y, eval_set=[(train_x, train_y)], eval_metric='auc', early_stopping_rounds=60)
    res['score'] = clf.predict_proba(test_x)[:, 1]
    res['score'] = res['score'].apply(lambda x: float('%.6f' % x))
    res.to_csv('./submissions/submission_model_add5_combine_leaf_5000_test2.csv', index=False)
    return clf

clf = LGB_train(train_x, train_y)

# get new feature and onehot

print("get new feature")
train_new = clf.apply(train_x)
test_new = clf.apply(test_x)
one = OneHotEncoder()
one.fit(train_new)
train_new = one.transform(train_new)
test_new = one.transform(test_new)

# combine with origin featurees

print("combine new feature")
# train_new = sparse.load_npz('./datasets/model_crossfea_origin_train_fea10_tree30.npz')
# test_new = sparse.load_npz('./datasets/model_crossfea_origin_test_fea10_tree30.npz')
X_train_new = sparse.hstack((train_x, train_new))
X_test_new = sparse.hstack((test_x, test_new))

print("saving new feature")
sparse.save_npz('./datasets/model_fea_add5_fea10_tree30_train_test2.npz', train_new)
sparse.save_npz('./datasets/model_fea_add5_fea10_tree30_test_test2.npz', test_new)
del train_new
del test_new
del train_x
del test_x
import gc
gc.collect()

# retrain and predict

# from sklearn.model_selection import train_test_split

# train_x_real, valid_x_real, train_y_real, valid_y_real = train_test_split(X_train_new, train_y, random_state=2018, test_size=0.3)

print('retrain the model with new feature...')

# X_train_new = sparse.load_npz('./datasets/model_crossfea_train.npz')
# X_test_new = sparse.load_npz('./datasets/model_crossfea_test.npz')
model = LGB_retrain(X_train_new, train_y, X_test_new, res)
joblib.dump(model, './model/model_add5_combine_leaf_5000_test2.model')

model = joblib.load('./datasets/model_combine_leaf_add5_5000.model')

res['score'] = model.predict_proba(X_test_new)[:, 1]
res['score'] = res['score'].apply(lambda x: float('%.6f' % x))
res.to_csv('./datasets/submission_model_combine_leaf_add5_5000_test2.csv', index=False)


from django.shortcuts import render, get_object_or_404
from django.http import HttpResponse, HttpRequest, JsonResponse, HttpResponseRedirect
from .models import Poll, Question, Choice, Vote, Answer, PollQuestion, ExtraData
from django.conf import settings
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.urls import reverse
from django.db.models import F
from django.utils import timezone
import json
from django_redis import get_redis_connection
from django.db.models import Prefetch
from django.contrib.auth.decorators import login_required
from django.template.loader import render_to_string
from django.contrib import messages
from django.core.cache import cache
from django.views.decorators.http import require_http_methods
import random
import datetime
import uuid

# redis连接池
con = get_redis_connection('default')

# Create your views here.

# class IndexView(generic.ListView):
#     model = Poll
#     template_name = 'polls/index.html'
#     context_object_name = 'polls'
#     paginate_by = getattr(settings, 'BASE_PAGE_BY', None)
#     paginate_orphans = getattr(settings, 'BASE_PAGE_ORPHANS', 0)

@login_required
def mine(request):
    user = request.user
    page_size = 8
    underway_polls = Poll.objects.filter(end_date__gt=timezone.now(), owner=user)
    finished_polls = Poll.objects.filter(end_date__lt=timezone.now(), owner=user)
    upolls_paginator = Paginator(underway_polls, page_size)
    fpolls_paginator = Paginator(finished_polls, page_size)
    upolls_page = request.GET.get('upage', 1)
    fpolls_page = request.GET.get('fpage', 1)
    upage = upolls_paginator.page(upolls_page)
    fpage = fpolls_paginator.page(fpolls_page)
    return render(request, 'jinja2/polls/mine.html', context=dict(
        upage=upage,
        fpage=fpage,
    ))

def index(request: HttpRequest):
    key = request.get_full_path()
    res = cache.get(key)

    underway_page = request.GET.get('underway_page')
    finished_page = request.GET.get('finished_page')
    if underway_page:
        # use cache
        if res:
            # print('use cache')
            return JsonResponse(res, safe=False)

        underway_polls = Poll.objects.filter(end_date__gt=timezone.now())
        paginator = Paginator(underway_polls, settings.BASE_PAGE_BY)

        try:
            underway_polls = paginator.page(underway_page)
        except PageNotAnInteger:
            underway_polls = paginator.page(1)
        except EmptyPage:
            underway_polls = paginator.page(paginator.num_pages)
        data = []
        for poll in underway_polls:
            data.append({'url': reverse('polls:detail', args=(poll.id,)), 'title': poll.title})

        cache.set(key, data, timeout=3600*24)
        return JsonResponse(data, safe=False)
    elif finished_page:
        # cache
        if res:
            # print('use cache')
            return JsonResponse(res, safe=False)

        finished_polls = Poll.objects.filter(end_date__lte=timezone.now())
        paginator = Paginator(finished_polls, settings.BASE_PAGE_BY)

        try:
            finished_polls = paginator.page(finished_page)
        except PageNotAnInteger:
            finished_polls = paginator.page(1)
        except EmptyPage:
            finished_polls = paginator.page(paginator.num_pages)

        data = []
        for poll in finished_polls:
            data.append({'url': reverse('polls:result', args=(poll.id,)), 'title': poll.title})

        cache.set(key, data, timeout=3600*24)
        return JsonResponse(data, safe=False)
    else:
        if res:
            # print('use cache')
            return render(request, 'jinja2/polls/index.html', res)
        underway_polls = Poll.objects.filter(end_date__gt=timezone.now())
        underway_polls_paginator = Paginator(underway_polls, settings.BASE_PAGE_BY)
        finished_polls = Poll.objects.filter(end_date__lte=timezone.now())
        finished_polls_paginator = Paginator(finished_polls, settings.BASE_PAGE_BY)
        first_underway_page_data = []  # 加入html
        for poll in underway_polls_paginator.page(1):
            first_underway_page_data.append({'url': reverse('polls:detail', args=(poll.id,)), 'title': poll.title})
        first_underway_page_data = json.dumps(first_underway_page_data)
        first_finished_page_data = []
        for poll in finished_polls_paginator.page(1):
            first_finished_page_data.append({'url': reverse('polls:result', args=(poll.id,)), 'title': poll.title})
        first_finished_page_data = json.dumps(first_finished_page_data)
        context = {'underway_polls': underway_polls_paginator.page(1),
                   'finished_polls': finished_polls_paginator.page(1),
                   'first_underway_page_data': first_underway_page_data,
                   'first_finished_page_data': first_finished_page_data}
        cache.set(key, context, timeout=3600*24)
        return render(request, 'jinja2/polls/index.html', context=context)

@login_required
def pub(request):
    questions = Question.objects.all()[:settings.BASE_PAGE_BY]
    choices = Choice.objects.all()[:settings.BASE_PAGE_BY]
    data = {}
    for q in questions:
        data.update({q.id: {'content': q.content, 'type': q.choice_type, 'index': 0}})
    questions_json = json.dumps(data)

    return render(request, 'jinja2/polls/pub.html', context={'question_list': questions,
                                                             'questions_json': questions_json,
                                                             'offset': settings.BASE_PAGE_BY})

def search_question_list(request):
    try:
        search = request.GET['search']
        if search:
            query_set = Question.objects.filter(content__contains=request.GET['search'])
        else:
            query_set = Question.objects.all()

        try:
            offset = int(request.GET['offset'])
        except KeyError:
            offset = 0

        questions = query_set[offset: offset + settings.BASE_PAGE_BY]

        if questions:
            data = {}
            for q in questions:
                data.update({q.id: {'content': q.content, 'type': q.choice_type, 'index': 0}})
            return JsonResponse({'questions': data, 'message': {'type': 'success', 'content': '请求成功'},
                                 'offset': offset + settings.BASE_PAGE_BY}, safe=False)
        else:
            return JsonResponse({'questions': {}, 'message': {'type': 'info', 'content': '没有跟多的数据'}})
    except KeyError:
        return JsonResponse({'questions': {}, 'message': {'type': 'error', 'content': '无效的请求'}}, safe=False)

def search_question_choices_list(request):
    try:
        search = request.GET['search']
        q_id = request.GET['q_id']
        if search:
            query_set = Choice.objects.filter(content__contains=search)
        else:
            if q_id:
                pq = PollQuestion.objects.filter(question__id=q_id).first()
                query_set = None
                if pq:
                    query_set = pq.choices.all()
                if not query_set:
                    query_set = Choice.objects.all()
            else:
                query_set = Choice.objects.all()

        try:
            offset = int(request.GET['offset'])
        except KeyError:
            offset = 0

        choices = query_set[offset: offset + settings.BASE_PAGE_BY]
        if choices:
            data = {}
            for c in choices:
                data.update({c.id: {'content': c.content, 'has_extra_data': 1 if c.has_extra_data else 0, 'index': 0}})
            return JsonResponse({'choices': data, 'message': {'type': 'success', 'content': '请求成功'},
                                 'offset': offset + settings.BASE_PAGE_BY}, safe=False)
        else:
            return JsonResponse({'message': {'type': 'info', 'content': '没有跟多的数据'}})
    except KeyError:
        return JsonResponse({'message': {'type': 'error', 'content': '无效的请求'}}, safe=False)

@require_http_methods(['GET', 'POST', 'DELETE'])
def detail(request: HttpRequest, poll_id):
    # key = request.path
    # 不能全部缓存，csrf_token会变化，jinja2片段缓存，env
    # html = con.get(key)
    # if html:
    #     return HttpResponse(html)
    if request.method == 'GET':
        if request.COOKIES.get('poll_%s' % poll_id):
            return HttpResponse('
您已经参与过给个调查了！
')
        vote_queryset = Vote.objects.select_related('choice').order_by('index')
        poll_questions_queryset = PollQuestion.objects.order_by('index').select_related('question').prefetch_related(
            Prefetch('votes', queryset=vote_queryset, to_attr='voteset'))
        poll = Poll.objects.prefetch_related(Prefetch('poll_questions',
                                                      queryset=poll_questions_queryset,
                                                      to_attr='pollquestions')).get(pk=poll_id)
        html = render_to_string('jinja2/polls/detail.html', {'poll': poll}, request=request)
        # con.set(key, html, ex=3600*24)
        return HttpResponse(html)
    elif request.method == 'POST':
        # 修改
        return HttpResponse('修改未实现')
    elif request.method == 'DELETE':
        # 删除
        poll = get_object_or_404(Poll, pk=poll_id)
        poll.delete()
        # pub数量减少，删除缓存
        cache.delete('/polls/')
        cache.delete_pattern('/polls/?*')
        return HttpResponse('删除成功')

def vote(request: HttpRequest, poll_id):
    # print(request.session.__dict__, 'session')
    # print({key: request.META[key] for key in request.META.keys() if key.startswith('HTTP')}, 'meta')

    post = request.POST
    poll = get_object_or_404(Poll, pk=poll_id)

    for k in post.keys():
        if k.startswith('csrf'):
            continue
        elif k.startswith('ex'):
            content = post.get(k)
            if content:
                _, vote_id = k.split(':')
                vote = get_object_or_404(Vote, pk=vote_id)
                extra_data = ExtraData(content=content, vote=vote)
                extra_data.save()
        elif k.startswith('pqq'):
            answer_content = post.get(k)
            if answer_content:
                _, pq_id = k.split(':')
                poll_question = get_object_or_404(PollQuestion, pk=pq_id)

                if request.user.is_authenticated:
                    answer = Answer(poll_question=poll_question, content=answer_content, user=request.user)
                else:
                    answer = Answer(poll_question=poll_question, content=answer_content)
                answer.save()
        elif k.startswith('pqc'):
            vote_id = post.get(k)
            vote = get_object_or_404(Vote, pk=vote_id)
            vote.num = F('num') + 1
            vote.save()
            if request.user.is_authenticated:
                vote.users.add(request.user)
    if poll.is_open:
        messages.add_message(request, messages.INFO, '谢谢参与')
        ret = HttpResponseRedirect(reverse('polls:result', args=(poll_id,)))
        ret.set_cookie('poll_%s' % poll.id, uuid.uuid4().hex, expires=poll.end_date)
        return ret
    else:
        return HttpResponse('
谢谢参与！
')

def result(request, poll_id):
    # print(request.session.__dict__, 'session')
    # print({key: request.META[key] for key in request.META.keys() if key.startswith('HTTP')}, 'meta')
    poll = get_object_or_404(Poll, pk=poll_id)
    if poll.is_open or (request.user and (request.user == poll.owner)):
        vote_queryset = Vote.objects.select_related('choice').order_by('index')
        poll_questions_queryset = PollQuestion.objects.order_by('index').select_related('question').prefetch_related(Prefetch('votes', queryset=vote_queryset, to_attr='voteset'))
        poll = Poll.objects.prefetch_related(Prefetch('poll_questions',
                                                      queryset=poll_questions_queryset,
                                                      to_attr='pollquestions')).get(pk=poll_id)
        return render(request, 'jinja2/polls/result.html', {'poll': poll, 'ExtraData': ExtraData})
    else:
        return HttpResponse('
该调查不公开结果！
')

@login_required
def preview(request: HttpRequest):

    data = json.loads(request.body)
    poll = Poll()
    try:
        poll.title = data['title']
        poll.description = data['desc']
        poll.owner = request.user
        poll.pollquestions = []
        for k in data['questions']:
            if k.startswith('new'):
                question = Question()
                question.choice_type = data['questions'][k]['type']
                question.content = data['questions'][k]['content']

            else:
                question = get_object_or_404(Question, pk=k)
            poll_question = PollQuestion(poll=poll, question=question, index=data['questions'][k]['index'])
            poll.pollquestions.append(poll_question)
            poll_question.voteset = []
            if question.choice_type == 1 or question.choice_type == 0:
                for j in data['choices'][k]:
                    c = data['choices'][k][j]
                    if j.startswith('new'):
                        choice = Choice(content=c['content'], has_extra_data=bool(c['has_extra_data']))
                    else:
                        choice = get_object_or_404(Choice, pk=j)
                    vote = Vote(poll_question=poll_question, choice=choice, index=c['index'])
                    poll_question.voteset.append(vote)

            poll_question.voteset.sort(key=lambda x: x.index)

        poll.pollquestions.sort(key=lambda x: x.index)
        messages.add_message(request, messages.INFO, '预览成功')
        return JsonResponse({'type': 'success', 'html': render_to_string('jinja2/polls/preview.html', {'poll': poll}, request=request)}, safe=False)
    except Exception as e:
        print(e)
        return JsonResponse({'type': 'error', 'info': '预览错误'})

@login_required
def submit(request: HttpRequest):
    # print(request.session.__dict__, 'session')
    # print(request.META, 'meta')
    data = json.loads(request.body)
    poll = Poll()
    try:
        poll.title = data['title']
        poll.description = data['desc']
        if data['end_date']:
            poll.end_date = datetime.datetime.strptime(data['end_date'], '%Y-%m-%d')
        if data['is_open'] == '0':
            poll.is_open = False
        poll.save()
        for k in data['questions']:
            if k.startswith('new'):
                question = Question()
                question.choice_type = data['questions'][k]['type']
                question.content = data['questions'][k]['content']
                question.save()
            else:
                question = get_object_or_404(Question, pk=k)
            poll_question = PollQuestion(poll=poll, question=question, index=data['questions'][k]['index'])
            poll_question.save()
            if question.choice_type == 1 or question.choice_type == 0:
                for j in data['choices'][k]:
                    c = data['choices'][k][j]
                    if j.startswith('new'):
                        choice = Choice(content=c['content'], has_extra_data=bool(c['has_extra_data']))
                        choice.save()
                    else:
                        choice = get_object_or_404(Choice, pk=j)
                    vote = Vote(poll_question=poll_question, choice=choice, index=c['index'])
                    vote.save()
        messages.add_message(request, messages.INFO, '提交成功')
        return JsonResponse({'type': 'success', 'redirect': reverse('polls:detail', args=(poll.id,))})
    except Exception as e:
        print(e)
        return JsonResponse({'type': 'error', 'info': '提交错误'})

@login_required
def gen_data(request):
    choices = []
    for i in range(5):
        for j in range(5):
            choice = Choice(content='-选项'+str(i)+str(j), has_extra_data=random.choice([True, False]))
            choice.save()
            choices.append(choice)

    questions = []
    for i in range(5):
        for j in range(5):
            question = Question(content='问题-{}{}?'.format(i, j), choice_type=random.randint(0, 2))
            question.save()
            questions.append(question)

    poll_questions = []
    pq_index = 0
    for i in range(35):
        poll = Poll(title='调查-{}'.format(i), description='描述-{}'.format(i))

        if random.randint(0, 1):
            poll.end_date = timezone.now()
        poll.save()
        index = 1
        qs = random.sample(questions, random.randint(3, 5))
        for q in qs:
            poll_questions.append([])
            poll_question = PollQuestion(poll=poll, question=q, index=index)
            index += 1
            poll_question.save()
            poll_questions[pq_index].append(poll_question)
            vote_index = 1
            votes = []
            if poll_question.question.choice_type != 2:
                cs = random.sample(choices, random.randint(3, 5))
                for c in cs:
                    vote = Vote(poll_question=poll_question, choice=c, index=vote_index)
                    vote_index += 1
                    vote.save()
                    votes.append(vote)
            poll_questions[pq_index].append(votes)
            pq_index += 1

    answer_index = 1
    extra_data_index = 1
    for ij in range(10):
        for pq, votes in poll_questions:
            if pq.question.choice_type == 2:
                answer = Answer(poll_question=pq, content='回答-{}'.format(answer_index))
                answer.save()
                answer_index += 1
            elif pq.question.choice_type == 1:
                    for vote in random.sample(votes, random.randint(1, len(votes))):
                        vote.num += 1
                        if vote.choice.has_extra_data:
                            extra_data = ExtraData(vote=vote, content='其他-{}'.format(extra_data_index))
                            extra_data.save()
                            extra_data_index += 1
                        vote.save()
            else:
                vote = random.choice(votes)
                vote.num += 1
                if vote.choice.has_extra_data:
                    extra_data = ExtraData(vote=vote, content='其他-{}'.format(extra_data_index))
                    extra_data.save()
                    extra_data_index += 1
                vote.save()

    return HttpResponse('test')

from haystack.views import SearchView

class PollSearchView(SearchView):
    template = 'jinja2/search/search.html'


from math import isclose
from typing import TYPE_CHECKING

from PyQt5.QtWidgets import QGraphicsPixmapItem
from PyQt5.QtGui import QTransform

from ..utils.expression import Condition, Expression

if TYPE_CHECKING:
    from typing import Optional, Any, Dict
    from PyQt5.QtGui import QPixmap

class ConditionGraphicsPixmapItem(QGraphicsPixmapItem):

    def __init__(self, condition: 'Optional[str]', *args: 'Any',
                 names: 'Optional[Dict[str, Any]]' = None,
                 **kwargs: 'Any') -> None:
        super().__init__(*args, **kwargs)

        self.__condition = Condition(condition) if condition else None
        self.__is_visible = True
        self.__condition_met = False
        self.__names = {} if names is None else names.copy()
        self.__x_offset_func: 'Optional[Expression]' = None
        self.__y_offset_func: 'Optional[Expression]' = None
        self.__x_offset_func_mul: float = 1
        self.__y_offset_func_mul: float = 1
        self.__x_offset_calc: float = 0
        self.__y_offset_calc: float = 0
        self.__angle_offset_func: 'Optional[Expression]' = None
        self.__angle_offset_calc: float = 0

        self.__pixmap = super().pixmap()

        self.evaluate()

    def show(self) -> None:
        self.setVisible(True)

    def hide(self) -> None:
        self.setVisible(False)

    def setVisible(self, visible: bool) -> None:
        super().setVisible(visible and self.__condition_met)
        self.__is_visible = visible

    def isVisible(self) -> bool:
        return self.__is_visible

    def setPixmap(self, pixmap: 'QPixmap') -> None:
        super().setPixmap(pixmap)

        self.__pixmap = pixmap

    def setOffset(self, *args: 'Any') -> None:
        super().setOffset(*args)

        if self.__x_offset_calc != 0 or self.__y_offset_calc != 0:

            offset = super().offset()

            super().setOffset(offset.x() + self.__x_offset_calc,
                              offset.y() + self.__y_offset_calc)

    def setXOffsetExpression(self, expression: 'Optional[str]',
                             multiplier: float = 1) -> None:

        if expression is None:
            self.__x_offset_func = None
        else:
            self.__x_offset_func = Expression(expression, default_value=0)

        self.__x_offset_func_mul = multiplier
        self.evaluate()

    def setYOffsetExpression(self, expression: 'Optional[str]',
                             multiplier: float = 1) -> None:

        if expression is None:
            self.__y_offset_func = None
        else:
            self.__y_offset_func = Expression(expression, default_value=0)

        self.__y_offset_func_mul = multiplier
        self.evaluate()

    def setAngleOffsetExpression(self, expression: 'Optional[str]') -> None:

        if expression is None:
            self.__angle_offset_func = None
        else:
            self.__angle_offset_func = Expression(expression, default_value=0)

        self.evaluate()

    def __updateOffset(self, new_calc_x: float, new_calc_y: float) -> None:

        offset = super().offset()

        super().setOffset(
            offset.x() - self.__x_offset_calc + new_calc_x,
            offset.y() - self.__y_offset_calc + new_calc_y)

        self.__x_offset_calc = new_calc_x
        self.__y_offset_calc = new_calc_y

    def evaluate(self, **kwargs: 'Any') -> None:

        self.__condition_met = self.__condition.evaluate(
            **self.__names, **kwargs) if self.__condition is not None else True

        if self.__is_visible:
            super().setVisible(self.__condition_met)

        offset_modif = False

        new_calc_x = 0
        if self.__x_offset_func is not None:
            new_calc_x = self.__x_offset_func.evaluate(
                **self.__names, **kwargs)*self.__x_offset_func_mul
            offset_modif = True

        new_calc_y = 0
        if self.__y_offset_func is not None:
            new_calc_y = self.__y_offset_func.evaluate(
                **self.__names, **kwargs)*self.__y_offset_func_mul
            offset_modif = True

        if offset_modif:
            self.__updateOffset(new_calc_x, new_calc_y)

        if self.__angle_offset_func is not None:
            new_angle = self.__angle_offset_func.evaluate(**self.__names,
                                                          **kwargs)

            if not isclose(self.__angle_offset_calc, new_angle,
                           rel_tol=0, abs_tol=0.5):

                self.__angle_offset_calc = new_angle
                super().setPixmap(self.__pixmap.transformed(
                    QTransform().rotate(new_angle)))


#!/usr/bin/env python3

from ardupi_weather.config import cfg, DATA_PATH, LOG_PATH
from ardupi_weather.arduino.arduino import arduino
from ardupi_weather.database import databaseController
import os
from ardupi_weather.terminal import terminal
from ardupi_weather.alarm import alarm
import datetime
import logging
from logging.handlers import TimedRotatingFileHandler
import time
import json

# Configuring the handler of the logger.
LOG_FILE = LOG_PATH + 'station.log'
handler = TimedRotatingFileHandler(LOG_FILE, when="midnight", backupCount=6)
handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s : %(message)s'))

# Configuring the logger.
logger = logging.getLogger('station')
logger.setLevel(logging.INFO)
logger.addHandler(handler)

def getTimestamp():

    """
    It returns the current timestamp of the machine.

    Returns:
        An integer containing the current timestamp.
    """

    return int(time.mktime(datetime.datetime.now().timetuple()))*1000

def formatDatetime(date):

    """
    It return the timestamp of a concret date.

    Args:
        date: A datetime object.

    Returns:
        An integer containing the timestamp of a concrete date.
    """

    return int(time.mktime(date.timetuple()))*1000

def printSensor(sensor, value):

    """
    It shows the current time, a sensor and its value in the terminal.

    Args:
        sensor: Sensor identifier to be printed
        value: The value for the sensor.
    """

    date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
    print (date, sensor, value)

class station:

    """
    Class dealing with the communications with an Arduino, processing of the data and its storing to a concrete
    database.

    It uses the information in config.yml to configure the station.
    """

    def __init__(self):

        """
        Initialization of the station. It includes the database and the Arduino.
        """

        self.dbc = databaseController.databaseController()
        self.newValueFunctions = []
        self.alarms = alarm()

        self.sensorData = cfg['arduino']['sensors']
        self.sensorNamesList = list(self.sensorData.keys())

        self.sensorTypes = cfg['data']['values']
        self.sensorUnits = {key: self.sensorTypes[value] for key,value in self.sensorData.items()}

        #Configure arduino
        arduinoCfg = cfg['arduino']
        self.configureArduino(arduinoCfg)

        #Configure database
        dataCfg = cfg['data']
        self.configureDatabase(dataCfg)

        #Initialize next updates database
        self.initializeNextUpdates()

        #Initialize history database
        historyCfg = dataCfg['history']
        self.initialitzeHistoryDatabase(historyCfg)

        #Initialize current data database
        currentDataCfg = dataCfg['currentData']
        self.initialitzeCurrentData(currentDataCfg)

        #Initialize daily history database
        dailyHistoryCfg = dataCfg['dailyHistory']
        self.initializeDailyHistoryDatabase(dailyHistoryCfg, historyCfg)

        #Initialize terminal input
        self.t = terminal()

    ###########################
    # Configuration functions #
    ###########################

    def configureArduino(self, ard):

        """
        It configures an Arduino using the specified connection and its options.

        Args:
            ard: Arduino configuration.
        """

        connection = ard['usedConnection']
        if connection == 'test':
            self.ard = arduino(connection, self.sensorUnits, self.sensorUnits)
        else:
            self.ard = arduino(connection, ard[connection], self.sensorUnits)

    def configureDatabase(self, data):

        """
        It configures the desired type of database (mongodb or sqlite).

        Args:
            data: Database configuration.
        """

        databaseName = data['name']

        usedDatabase = data['usedDB']
        db = data[usedDatabase]
        if usedDatabase == 'sqlite':
            self.dbc.enableSqlite(db['path'] + databaseName)
        elif usedDatabase == 'mongo':
            self.dbc.enableMongo(databaseName, db['uri'], DATA_PATH, LOG_PATH)
        else:
            'Unknown database name'

    def initializeNextUpdates(self):

        """
        It configures the "nextUpdates" container. It has the next update time for currentData,
        history and dailyHistory containers.

        This information is used to make possible to the browsers to autoupdate the webpage data
        without refreshing the it.
        """

        self.dbc.createContainer('nextUpdates', {'name':'TEXT PRIMARY KEY','nextUpdate':'TEXT'})

    def initialitzeCurrentData(self, currentData):

        """
        It initializes the current data container of the database.

        Args:
            currentData: Current data configuration.
        """

        if currentData['enable']:
            self.currentDataDatabaseName = currentData['name']
            self.dbCurrentDataHeader = {'type':'TEXT PRIMARY KEY', 'value': 'TEXT'}

            self.currentDataValues = {}

            self.dbc.createContainer(self.currentDataDatabaseName, self.dbCurrentDataHeader)

            # Initialize current data database
            for key in self.sensorTypes.keys():
                self.dbc.upsert(self.currentDataDatabaseName, 'type', {'type':key, 'value':None})

            # Add update current data alarm
            update = currentData['updateEvery']
            self.alarms.add('updateCurrentData', self.updateCurrentDataDatabase, update['d'], update['h'], update['m'], update['s'])

            # Insert next update to database
            self.dbc.upsert('nextUpdates', 'name',  {'name':'currentData','nextUpdate': self.alarms.getNextUpdateStr('updateCurrentData')})

            self.newValueFunctions.append(self.updateCurrentData)

    def initialitzeHistoryDatabase(self, history):

        """
        It initializes the history container of the database.

        Args:
            history: History configuration.
        """

        if history['enable']:
            self.dbHistoryHeader = {'date' : 'TIMESTAMP'}
            self.dbHistoryHeader.update(self.sensorTypes)

            self.historyDataName = history['name']
            self.dbc.createContainer(self.historyDataName, self.dbHistoryHeader)

            self.sensorSum = dict.fromkeys(self.sensorTypes, 0)
            self.sensorNum = dict.fromkeys(self.sensorTypes, 0)

            update = history['updateEvery']
            self.alarms.add('updateHistory', self.updateHistoryDatabase, update['d'], update['h'], update['m'], update['s'])

            self.dbc.upsert('nextUpdates', 'name', {'name':'history','nextUpdate': self.alarms.getNextUpdateStr('updateHistory')})

            self.newValueFunctions.append(self.updateHistoryData)

    def initializeDailyHistoryDatabase(self, daily, history):

        """
        It initializes the daily history container of the database.

        Args:
            daily: Daily history configuration.
            history: History configuration.
        """

        if daily['enable']:
            if not history['enable']:
                print ("Daily history cannot be enabled because 'history' is not enabled")
                return

            self.dbDailyHistoryHeader = {'date' : 'TEXT'}
            self.dbDailyHistoryHeader.update({key + 'MAX': value for key,value in self.sensorTypes.items()})
            self.dbDailyHistoryHeader.update({key + 'MIN': value for key,value in self.sensorTypes.items()})
            self.dbDailyHistoryHeader.update({key + 'AVG': value for key,value in self.sensorTypes.items()})

            self.dailyHistoryDataName = daily['name']
            self.dbc.createContainer(self.dailyHistoryDataName, self.dbDailyHistoryHeader)

            self.alarms.addDaily('updateDailyHistory', self.updateDailyHistoryDatabase)

            self.dbc.upsert('nextUpdates', 'name',  {'name':'dailyHistory','nextUpdate': self.alarms.getNextUpdateStr('updateDailyHistory')})

    ########################################################
    # Functions to execute when a new sensor value arrives #
    ########################################################

    def updateCurrentData(self, valueType, value):

        """
        It updates the current value of a type of data.

        Args:
            valueType: The type of data to update.
            value: The new value.
        """

        self.currentDataValues[valueType] = value

    def updateHistoryData(self, valueType, value):

        """
        It updates intermediate history data.

        Args:
            valueType: The type of data to update.
            value: The new value.
        """

        self.sensorSum[valueType] += value
        self.sensorNum[valueType] += 1

    #############################################
    # Functions to execute at a determined time #
    #############################################

    def updateCurrentDataDatabase(self):

        """
        It updates the data of the current data database and the next updates database.
        """

        for (key,value) in self.currentDataValues.items():
            self.dbc.update(self.currentDataDatabaseName,{"value":value}, "type", key)

        # Change next update time
        nextUpdate = self.alarms.getNextUpdateStr('updateCurrentData')
        self.dbc.update("nextUpdates", {"nextUpdate":nextUpdate}, "name", "currentData")

        print('Updated current data file')

    def updateHistoryDatabase(self):

        """
        It updates the history database and the next updates database.

        The data stored in it is the mean of the received values from the last update to the moment
        this function is called.
        """

        # Compute means from the received values.
        newValues = {'date' : getTimestamp()}
        nullKeys = []
        for elem in self.sensorSum.keys():
            if self.sensorNum[elem] == 0:
                newValues[elem] = None
                nullKeys.append(elem)
            else:
                newValues[elem]= round(self.sensorSum[elem]/self.sensorNum[elem], 1)

        if len(nullKeys) != 0:
            logger.warning('-History update- No data received for %s', ', '.join(nullKeys))

        logger.info('-History update- %s', ', '.join("%s:%i" % (k,v) for k,v in self.sensorNum.items()))

        # Update database
        self.dbc.insert(self.historyDataName, newValues)

        # Change next update time
        nextUpdate = self.alarms.getNextUpdateStr('updateHistory')
        self.dbc.update("nextUpdates", {"nextUpdate":nextUpdate}, "name", "history")

        print('Inserted data to history')

        # Reset data
        self.sensorSum = dict.fromkeys(self.sensorSum, 0)
        self.sensorNum = dict.fromkeys(self.sensorNum, 0)

    def updateDailyHistoryDatabase(self):

        """
        It updates the daily history database and the next updates database.

        The mean, minimum and maximum values are computed from the history database. It means that usually
        the values of the minimum and maximum values will not be the correct ones, but an approximation.
        Using smaller updating time for the history involve having smaller errors in the daily history.
        """

        # Query to history database
        lastDay = datetime.date.today() - datetime.timedelta(days=1)
        minVal = datetime.datetime.combine(lastDay, datetime.datetime.min.time())
        maxVal = datetime.datetime.combine(lastDay, datetime.datetime.max.time())
        data = self.dbc.queryBetweenValues(self.historyDataName, 'date', formatDatetime(minVal), formatDatetime(maxVal))
        dailyData = dict.fromkeys(self.dbDailyHistoryHeader, 0)
        dailyData['date'] = formatDatetime(lastDay)
        numValues = dict.fromkeys(self.sensorTypes, 0)

        # Initialization of dailyData
        for column in dailyData:
            if 'MIN' in column:
                dailyData[column] = float('inf')
            elif 'MAX' in column:
                dailyData[column] = float('-inf')
            elif 'AVG' in column:
                dailyData[column] = 0.

        # Calculus of MIN, MAX and addition of all values to AVG
        for entry in data:
            for key,value in entry.items():
                if key != 'date' and value != None:
                    dailyData[key + 'MIN'] = min(dailyData[key + 'MIN'], value)
                    dailyData[key + 'MAX'] = max(dailyData[key + 'MAX'], value)
                    dailyData[key + 'AVG'] += value
                    numValues[key] += 1

        # Calculus of AVG
        for key in self.sensorTypes.keys():
            if numValues[key] != 0:
                dailyData[key + 'AVG'] = round(dailyData[key + 'AVG']/numValues[key],1)
            else:
                # there is no value registered for this sensor in 'lastDay'
                dailyData[key + 'AVG'] = None
                dailyData[key + 'MIN'] = None
                dailyData[key + 'MAX'] = None

        # Update daily history database.
        self.dbc.insert(self.dailyHistoryDataName, dailyData)

        # Change next update time
        nextUpdate = self.alarms.getNextUpdateStr('updateDailyHistory')
        self.dbc.update("nextUpdates", {"nextUpdate":nextUpdate}, "name", "dailyHistory")

        print('Inserted data to daily history')

    ###############
    # Run station #
    ###############

    def run(self):

        """
        This function starts the station, from this moment it will receive data from the Arduino and store
        it to the database.
        """

        while True:

            # Read data from Arduino and update the partial data
            data = self.ard.readInput()

            # Execute functions when it gets new data
            for (sensor,value) in data:
                if sensor in self.sensorNamesList:
                    valueType = self.sensorData[sensor]
                    for f in self.newValueFunctions:
                        f(valueType, value)
                    printSensor(sensor, value)

            # Execute functions when it is necessary
            toDo = self.alarms.getThingsToDo()
            for func in toDo:
                func()

            # Read from terminal
            inp = self.t.readLine()
            if inp != "":
                self.ard.write(inp)

            time.sleep(0.1)

def main():
    s = station()
    s.run()

if __name__ == '__main__':
    s = station()
    s.run()


import json

import pytest
import responses

from abusor.client.abusor_api_client import parse_arguments, post_to_api

try:
    from django.urls import reverse
except ImportError:
    # for django < 1.10
    from django.core.urlresolvers import reverse

def test_post_to_api(fake, mock_resp):
    """Verify that the data is posted correctly to the Abusor API."""
    uri = fake.url().rstrip("/")
    token = fake.sha1()
    ip = fake.ipv4()
    subject = fake.sentence()

    # prepare the mock response
    endpoint = uri + reverse("event-list")
    mock_resp.add(responses.POST, endpoint, status=201)

    result = post_to_api(uri, token, ip, subject)
    assert result is None

    # inspect the performed request
    req = mock_resp.calls[-1].request
    assert "Authorization" in req.headers
    assert req.headers["Authorization"] == "Token {}".format(token)

    payload = json.loads(req.body.decode())
    assert "ip_address" in payload
    assert payload["ip_address"] == ip
    assert "subject" in payload
    assert payload["subject"] == subject

def test_post_to_api_error(fake, mock_resp):
    """Verify that an API error is reported correctly."""
    uri = fake.uri().rstrip("/")
    token = None
    ip = fake.ipv4()
    subject = fake.sentence()

    # prepare the mock response
    endpoint = uri + reverse("event-list")
    mock_resp.add(responses.POST, endpoint, body="No unauthorized access", status=401)

    with pytest.raises(SystemExit) as excinfo:
        post_to_api(uri, token, ip, subject)
    assert "[401]" in str(excinfo.value)
    assert "No unauthorized access" in str(excinfo.value)

def test_parse_arguments_subject_escaping(fake, monkeypatch):
    """Verify that subject escaping using double underscore works properly."""
    ip = fake.ipv4()
    subject = fake.sentence()
    assert " " in subject

    escaped_subject = subject.replace(" ", "__")
    assert " " not in escaped_subject

    monkeypatch.setattr(
        "sys.argv", ["progname", "--subject", escaped_subject, "--ip", ip]
    )
    args = parse_arguments()
    assert args.subject == subject

def test_parse_arguments_environment_vars(fake, monkeypatch):
    """Verify that we can set various arguments using environment vars."""
    uri = fake.uri().rstrip("/")
    token = fake.sha1()
    monkeypatch.setenv("ABUSOR_URI", uri)
    monkeypatch.setenv("ABUSOR_TOKEN", token)

    monkeypatch.setattr("sys.argv", ["progname", "-i", "1.2.3.4", "-s", "foo"])
    args = parse_arguments()
    assert args.uri == uri
    assert args.token == token


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
    File Name:     req
    Author:        Administrator
    Date:          2018/7/20
    Description:
# code is far away from bugs with the god animal protecting
    I love animals. They taste delicious.
              ┏┓      ┏┓
            ┏┛┻━━━┛┻┓
            ┃      ☃       ┃
            ┃  ┳┛  ┗┳  ┃
            ┃      ┻      ┃
            ┗━┓      ┏━┛
                ┃      ┗━━━┓
                ┃  神兽保佑    ┣┓
                ┃  永无BUG！   ┏┛
                ┗┓┓┏━┳┓┏┛
                  ┃┫┫  ┃┫┫
                  ┗┻┛  ┗┻┛
"""
__author__ = 'Administrator'
import requests
session = requests.session()

def get_result(url,data={},header={}):
    result = session.get(url,data=data,headers=header)
    print('url:  %s    ===================================================================================' % url)
    # print(result.status_code,result.text)
   # assert int(result.status_code) < 400
    return result.text,result.status_code

def post_result(url,data={}):
    result = session.post(url,data)
    print('url:  %s    ===================================================================================' % url)
    print(result.status_code, result.text)
    # assert int(result.status_code) < 400
    return result.text

def post_file(url,data,files={}):
    print('url:  %s    ===================================================================================' % url)
    result = session.request('post', url,
            params=None, data=data, headers=None, cookies=None, files=files,
            auth=None, timeout=5, allow_redirects=True, proxies=None,
            hooks=None, stream=None, verify=None, cert=None, json=None)
    print(result.status_code, result.text)
    # assert int(result.status_code) < 400
    return result.text

def get_download(url,data,fn):
    if not fn:
        print("文件名不能为空")
    print('url:  %s    ===================================================================================' % url)
    result = session.get(url,data=data)
    print(result.status_code,'download file %s succeed!')
    if int(result.status_code) < 400:
        ct = result.content
        with open(fn,'wb') as file:
            file.write(ct)

if __name__ == "__main__":
    ret,code = get_result('http://www.360doc.com/content/13/0722/22/998704_301833252.shtml')
    # print(ret)
    txt = ret[ret.find('旧历的'):ret.find('二月七日')+4]
    with open('../data/祥林嫂.txt','w',encoding='utf8') as tx:
        tx.write(txt)


import time

from selenium import webdriver
from selenium.webdriver import ActionChains

driver = webdriver.Chrome()
driver.implicitly_wait(10)
driver.get('http://www.baidu.com')

# 复杂版
# 初始化 ActionChains:动作链条
action = ActionChains(driver)
# 定位一个元素
elem = driver.find_element('xpath',"//span[@id='s-usersetting-top']")
# click操作
action.click(elem).perform()
time.sleep(5)

# 简单版
# h2 = driver.find_element('xpath','//h2')
# h2.click()


import csv
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

data=list(csv.reader(open('spx_prices.csv')))
del(data[0])
data_prices=[]
for line in data:
    line[0],line[1]=int(line[0]),float(line[1])
    data_prices.append(line[1])

mean_tab=[]
var_tab=[]
min_tab=[]
max_tab=[]
best_10=[]
cum_26=[]
cum_28=[]

#Testuje strategie dla signal_window od 2 do 99, best_10 bedzie tabelka na 10 najlepszych wynikow
#patrzac na sredni profit w latach 2004-2014.

for n in range(2,100):

    signal=0
    signal_window=n
    position=0
    max_position=100
    cum_pnl=[0.0]

    for i in range(100,len(data)):
        pnl_daily=0

        if data[i][1]>max(data_prices[i-signal_window:i]):
            signal=1
        elif data[i][1]<min(data_prices[i-signal_window:i]):
                signal=-1

        pnl_daily=50*position*(data[i][1]-data[i-1][1])
        cum_pnl.append(cum_pnl[-1]+pnl_daily)
        position=max_position*signal

    if n==26:
        cum_26=cum_pnl
    if n==28:
        cum_28=cum_pnl

    mean_tab.append(np.mean(cum_pnl))
    var_tab.append(np.var(cum_pnl))
    min_tab.append(round(min(cum_pnl),2))
    max_tab.append(round(max(cum_pnl),2))

    if len(best_10)<10:
        best_10.append(np.mean(cum_pnl))
    else:
        if np.mean(cum_pnl)>min(best_10):
            for i,x in enumerate(best_10):
                if x==min(best_10):
                    del best_10[i]
                    best_10.append(np.mean(cum_pnl))
                    break
#Teraz ogarniam statystyki dla 10 najlepszych wynikow do dataframe'a
col_n=[]
col_mean=[]
col_var=[]
col_min=[]
col_max=[]

for i,x in enumerate(mean_tab):
    for el in best_10:
        if x==el :
            col_n.append(i+2)
            col_mean.append(round(mean_tab[i],2))
            col_var.append(round(var_tab[i]))
            col_min.append(min_tab[i])
            col_max.append(max_tab[i])
            break

#Dodaje sobie jeszcze wiersz ze srednimi wartosciami dla wszystkich parametrow signal_window

col_n.append('avg')
col_mean.append(round(np.mean(mean_tab),2))
col_var.append(round(np.mean(var_tab)))
col_min.append(round(np.mean(min_tab)))
col_max.append(round(np.mean(max_tab)))

data={'Mean':col_mean,'Variance':col_var,'Min':col_min, 'Max':col_max}
df=pd.DataFrame(data,index=(col_n),columns=('Mean','Variance','Min','Max'))
print(df)

#Zatem najlepszy wynik strategia osiagnela dla signal_window=26, dla 28 byla najstabilniejsza, wiec sygnal
#generowany dl takiej wartosci byl najpewniejszy. Srednio przynosila strate prawie 670 000.

#Jeszcze widac ze strategia bylaby srednio oplacalna gdyby nie jedna seria spadkow miedzy 1000 a 1500
#dniem z pobranych danych

plt.plot(cum_26)
plt.plot(cum_28)

plt.show()


# -*- coding:utf-8 -*-

test_mode = 'pillow'

"""pillow로 했는데 여기서 암튼 뭐때문에 하이패스 필터 -> 이미지가 날카로워짐."""

if(test_mode=='pillow'):
    from PIL import Image, ImageFilter
    image=Image.open("9D194DF8-01FE-4E4E-8AE0-522C77143077.jpg")
    #image.show()
    #image.rotate(45).show()
    im_blur = image.filter(filter=ImageFilter.EDGE_ENHANCE)
        #BLUR, CONTOUR, DETAIL, EDGE_ENHANCE, EMBOSS, SHARPEN, SMOOTH, SMOOTH_MORE
    im_blur.show()
    im_blur.save("pillow_sangwa.jpg")

"""근데 pillow로 실행한 이미지 파일은 ndarray가 아니라서 아까처럼 우리가 직접만든 필터로 할 수 없음!!"""

"""pillow에서 ndarray로 캐스트하는 방법을 찾아야함!"""


#!/usr/bin/python3

#Show a slider widget

import sys
from PyQt5.QtWidgets import QWidget, QSlider, QLabel, QApplication
from PyQt5.QtGui import QPixmap
from PyQt5.QtCore import Qt

class Example(QWidget):

    def __init__(self):
        super().__init__()
        self.initUI()

    def initUI(self):

        #create horizontal slider
        sld = QSlider(Qt.Horizontal, self)
        sld. setFocusPolicy(Qt.NoFocus)
        sld.setGeometry(30, 40, 100, 30)
        #connect integer version of valueChanged signal to our defined function changeValue
        sld.valueChanged[int].connect(self.changeValue)

        #create label and add an image to it
        self.label = QLabel(self)
        self.label.setPixmap(QPixmap('mute.png'))
        self.label.setGeometry(160, 40, 80, 30)

        self.setGeometry(300, 300, 280, 170)
        self.setWindowTitle('Slider')

    #based on the value of the slider we set a different image in the label
    def changeValue(self, value):

        if value == 0:
            self.label.setPixmap(QPixmap('mute.png'))
        elif value > 0 and value <= 30:
            self.label.setPixmap(QPixmap('min.png'))
        elif value > 30 and value <= 80:
            self.label.setPixmap(QPixmap('med.png'))
        else:
            self.label.setPixmap(QPixmap('max.png'))

if __name__ == '__main__':
    app = QApplication(sys.argv)
    ex = Example()
    ex.show()
    sys.exit(app.exec_())


#!/usr/bin/env python
from utils.decorators import timeit

def find_digits(digits_left, goal):
    if (digits_left, goal) not in find_digits.cache:
        if not digits_left:
            find_digits.cache[(0, goal)] = (goal == 0, 0)
        else:
            count = total = 0
            for start in filter(lambda n: pow(n, 2) <= goal, range(10)):
                subcount, subtotal = find_digits(digits_left - 1, goal - start ** 2)
                count += subcount
                total += subtotal + (start * 10 ** (digits_left - 1)) * subcount
            find_digits.cache[(digits_left, goal)] = (count, total)
    return find_digits.cache[(digits_left, goal)]

find_digits.cache = {}

@timeit
def solve_it():
    return sum(find_digits(20, pow(n, 2))[1] for n in range(40)) % 10 ** 9

def main():
    answer, time = solve_it()
    print("Solution is --- {}\nRunning time --- {:2.2f} seconds".format(answer, time))

if __name__ == "__main__":
    main()


from http import HTTPStatus
import time

from django.test import TestCase, Client
from django.core.cache import cache

from ..models import Post, Group, User

class PostsURLTests(TestCase):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.user = User.objects.create_user(username='user')
        cls.author = User.objects.create_user(username='author')
        cls.group = Group.objects.create(
            title='Тестовая группа',
            slug='test-slug',
        )
        cls.post = Post.objects.create(
            author=cls.author,
            text='Тестовый пост',
        )

        cls.url_index = '/'
        cls.url_group_list = f'/group/{cls.group.slug}/'
        cls.url_profile = f'/profile/{cls.user}/'
        cls.url_post_detail = f'/posts/{cls.post.pk}/'
        cls.url_post_create = '/create/'
        cls.url_post_edit = f'/posts/{cls.post.pk}/edit/'
        cls.url_404 = '/non-existing-page/'

        cls.public_urls = (
            cls.url_index,
            cls.url_group_list,
            cls.url_profile,
            cls.url_post_detail
        )

        cls.private_urls = (
            cls.url_post_create,
            cls.url_post_edit,
        )

        cls.urls_template = [
            (cls.url_index, 'posts/index.html'),
            (cls.url_group_list, 'posts/group_list.html'),
            (cls.url_profile, 'posts/profile.html'),
            (cls.url_post_detail, 'posts/post_detail.html'),
            (cls.url_post_create, 'posts/create_post.html'),
            (cls.url_post_edit, 'posts/create_post.html'),
            (cls.url_404, 'core/404.html')
        ]

    def setUp(self):
        cache.clear()

        self.guest_client = Client()

        self.authorized_client = Client()
        self.authorized_client.force_login(PostsURLTests.user)

        self.author_client = Client()
        self.author_client.force_login(PostsURLTests.author)

    def test_urls_public(self):
        """Проверяем доступность общедоступных страниц"""
        for url in PostsURLTests.public_urls:
            with self.subTest(url=url):
                response = self.guest_client.get(url)
                self.assertEqual(response.status_code, HTTPStatus.OK)

    def test_urls_private(self):
        """Проверяем доступность приватных страниц"""
        for url in PostsURLTests.private_urls:
            with self.subTest(url=url):
                response = self.author_client.get(url)
                self.assertEqual(response.status_code, HTTPStatus.OK)

    def test_edit_urls_redirect_not_author(self):
        """Проверяем редирект с /posts/<post_id>/edit/ для не авторов поста"""
        response = self.authorized_client.get(PostsURLTests.url_post_edit)
        self.assertRedirects(response, PostsURLTests.url_post_detail)

    def test_urls_redirect_anonymous(self):
        """Проверяем редиректы для неавторизованного пользователя"""
        templates_url_names = [
            (url, f'/auth/login/?next={url}')
            for url in PostsURLTests.private_urls
        ]
        for url, status in templates_url_names:
            with self.subTest(url=url):
                response = self.guest_client.get(url)
                self.assertRedirects(response, status)

    def test_edit_urls_exists_at_desired_location_authorized_author(self):
        """Проверяем доступность /posts/<post_id>/edit/ для автора"""
        response = self.author_client.get(PostsURLTests.url_post_edit)
        self.assertEqual(response.status_code, HTTPStatus.OK)

    def test_urls_uses_correct_template(self):
        """URL-адрес использует соответствующий шаблон."""
        for url, template in PostsURLTests.urls_template:
            with self.subTest(url=url):
                response = self.author_client.get(url)
                self.assertTemplateUsed(response, template)


from django.db.models.signals import post_save
from django.dispatch import receiver

from tenants.models import (
    Tenant,
    Organization,
    OrganizationMembership
)
from tenants.services import OrganizationService

@receiver(post_save, sender=Tenant)
def create_playground_organization(sender, instance=None, created=False, **kwargs):
    if created:
        org = Organization.objects.create(
            name='Playground',
            is_playground=True
        )
        OrganizationMembership.objects.create(
            tenant=instance,
            organization=org,
            is_owner=True,
            is_admin=True
        )

@receiver(post_save, sender=Organization)
def bootstrap_organization(sender, instance=None, created=False, **kwargs):
    if created:
        OrganizationService.bootstrap_organization(organization=instance)


import requests
import pandas as pd

zipcode_data = pd.read_csv(
    'D:\\...',  dtype=str)

tdbank_data = pd.DataFrame(columns=[
    'ID', 'Type', 'Address', 'City', 'State', 'Latitude', 'Longitude', 'Zipcode'])

URL = "https://locations.td.com/index.html"

try:
    for index in range(len(zipcode_data)):
        zipcode = zipcode_data.iloc[index]['ZIP_CODE']
        res = requests.get(
            url=URL, params={'q': zipcode}, headers={'Accept': 'application/json'})
        data = res.json()
        stores = data['response']['entities']

        if len(stores) != 0:
            for store in range(len(stores)):
                ID = stores[store]['distance']['id']
                try:
                    loc_type = stores[store]['profile']['c_locationTypeName']
                except:
                    loc_type = None
                addr1 = stores[store]['profile']['address']['line1']
                addr2 = stores[store]['profile']['address']['line2']
                addr3 = stores[store]['profile']['address']['line3']
                address = ''
                if addr1 != None:
                    address = address + addr1
                if addr2 != None:
                    address = address + addr2
                if addr3 != None:
                    address = address + addr3
                city = stores[store]['profile']['address']['city']
                state = stores[store]['profile']['address']['region']
                try:
                    latitude = stores[store]['profile']['geocodedCoordinate']['lat']
                    longitude = stores[store]['profile']['geocodedCoordinate']['long']
                except:
                    latitude = None
                    longitude = None

                if (ID in tdbank_data['ID'].values):
                    continue
                else:
                    tdbank_data = tdbank_data.append(
                        {'ID': ID, 'Type': loc_type, 'Address': address, 'City': city, 'State': state, 'Latitude': latitude, 'Longitude': longitude, 'Zipcode': zipcode}, ignore_index=True)

finally:
    tdbank_data.to_csv(r'D:\\...',
                       index=False, header=True)


import django_filters
from django_filters import FilterSet, DateTimeFilter
from django.forms import DateTimeInput
from .models import Post

class PostFilter(FilterSet):
    title = django_filters.CharFilter(
        field_name='title',
        lookup_expr='icontains',
        label='Заголовок',
    )

    type = django_filters.ChoiceFilter(
        field_name='category_type',
        label='Тип',
        empty_label='Выбрать',
        choices=Post.CATEGORY_CHOICES,
    )

    added_after = DateTimeFilter(
        field_name='date_create',
        lookup_expr='gt',
        label='Дата публикации с:',
        widget=DateTimeInput(
            format='%Y-%m-%dT%H:%M',
            attrs={'type': 'datetime-local'},
        )
    )


import json
import sys
import logging
from forms import *
from config import db, app, migrate
from flask import (
    abort,
    flash,
    redirect,
    render_template,
    request,
    url_for,
)
from forms import VenueForm
from model import Venue, Show, Artist
from filters import format_datetime

#  Venues
#  ----------------------------------------------------------------

@app.route('/venues')
def venues():
  # Done: num_shows should be aggregated based on number of upcoming shows per venue.(seem Done)

  #...  not dynamic with events
  #...  define an empty lists to store the data
  city_and_state = []
  data = []

  #... all venus on the same city and state
  venue_table = Venue.query.all()
  #... itrate on all rows in the query
  for row in venue_table:
    if city_and_state == row.city + row.state:
      data[len(data) - 1]["venues"].append({
        "id": row.id,
        "name": row.name,
        "upcoming_shows_count": row.upcoming_shows_count
      })

    else:
      city_and_state = row.city + row.state
      data.append({"city": row.city, "state": row.state,
          "venues": [{"id": row.id, "name": row.name,
          "upcoming_shows_count": row.upcoming_shows_count }]
          })

  return render_template('pages/venues.html', areas=data)

@app.route('/venues/search', methods=['POST'])
def search_venues():
  # DONE: implement search on artists with partial string search. Ensure it is case-insensitive.(one Bug)
  # seach for Hop should return "The Musical Hop".
  # search for "Music" should return "The Musical Hop" and "Park Square Live Music & Coffee"

  # ilike operator instead of like operator to match case insensitive.
  search_term = request.form.get("search_term", "")
  venues = Venue.query.filter(Venue.name.ilike(f"%{search_term}%")).all()
  now = datetime.now()
  venue_list = []
  for venue in venues:
      venue_shows = Show.query.filter_by(venue_id=venue.id).all()
      num_upcoming = 0
      for show in venue_shows:
          if show.start_time > now:
              num_upcoming += 1

      venue_list.append({
        "id": venue.id,
        "name": venue.name,
        "num_upcoming_shows": num_upcoming
      })

  response={
    "count": len(venues),
    "data": venue_list
  }
  return render_template('pages/search_venues.html', results=response, search_term=search_term)

@app.route('/venues/<int:venue_id>')
def show_venue(venue_id):
  # shows the venue page with the given venue_id
  # DONE: replace with real venue data from the venues table, using venue_id (2 bugs)

  venue = Venue.query.get(venue_id)
  if venue:
    data = venue.venue_to_dictionary()
    data={
      "id": venue.id,
      "name": venue.name,
      "genres": [venue.genres],
      "address": venue.address,
      "city": venue.city,
      "state": venue.state,
      "phone": venue.phone,
      "website": venue.web_site,
      "facebook_link": venue.facebook_link,
      "seeking_talent": True,
      "seeking_description": venue.seeking_description,
      "image_link": venue.image_link,
    }

    current_time = datetime.now().strftime('%Y-%m-%d')

    new_shows_query = Show.query.options(db.joinedload('artist')).filter(Show.venue_id == venue_id).filter(Show.start_time > current_time).all()
    new_show = list(map(Show.artist_details, new_shows_query))
    data["upcoming_shows"] = new_show
    data["upcoming_shows_count"] = len(new_show)
    past_shows_query = Show.query.options(db.joinedload('artist')).filter(Show.venue_id == venue_id).filter(Show.start_time <= current_time).all()
    past_shows = list(map(Show.artist_details, past_shows_query))
    data["past_shows"] = past_shows
    data["past_shows_count"] = len(past_shows)

    return render_template('pages/show_venue.html', venue=data)
  else:
    return render_template('errors/404.html')

@app.route('/venues/<int:venue_id>/edit', methods=['GET'])
def edit_venue(venue_id):
  venue = Venue.query.get(venue_id)
  if venue:
    form = VenueForm(obj=venue)
    venue={
      "id": venue_id,
      "name": venue.name,
      "genres": [venue.genres],
      "address": venue.address,
      "city": venue.city,
      "state": venue.city,
      "phone": venue.phone,
      "website": venue.web_site,
      "facebook_link": venue.facebook_link,
      "seeking_talent": venue.seeking_talent,
      "seeking_description": venue.seeking_description,
      "image_link": venue.image_link
    }
    # DONE: populate form with values from venue with ID <venue_id>
    return render_template('forms/edit_venue.html', form=form, venue=venue)

  else:
    flash('An error occurred. Artist ' + request.form['name'] + ' could not be edited.')
    return redirect(url_for('index'))

@app.route('/venues/<int:venue_id>/edit', methods=['POST'])
def edit_venue_submission(venue_id):
  # LASTXX: take values from the form submitted, and update existing
  # venue record with ID <venue_id> using the new attributes
  form = VenueForm()
  name = form.name.data
  city = form.city.data
  state = form.state.data
  address = form.address.data
  phone = form.phone.data
  genres = form.genres.data
  facebook_link = form.facebook_link.data
  image_link = form.image_link.data
  web_site = form.web_site.data
  seeking_talent = True if form.seeking_talent.data == 'Yes' else False
  seeking_description = form.seeking_description.data

  if not form.validate():
    flash( form.errors )
    return redirect(url_for('edit_venue_submission', venue_id=venue_id))

  else:
    error_in_update = False
    try:
      venue = Venue.query.get(venue_id)
      venue.name = name
      venue.city = city
      venue.state = state
      venue.address = address
      venue.phone = phone
      venue.seeking_talent = seeking_talent
      venue.seeking_description = seeking_description
      venue.image_link = image_link
      venue.web_site = web_site
      venue.facebook_link = facebook_link
      venue.genres = []
      for genre in genres:
        venue.genres.append(genre)

      db.session.commit()
    except Exception:
      error_in_update = True
      print(f'Exception "{Exception}" in edit_venue_submission()')
      db.session.rollback()
    finally:
      db.session.close()

    if not error_in_update:
      # on successful db update, flash success
      flash('Venue ' + request.form['name'] + ' was successfully updated!')
      return redirect(url_for('show_venue', venue_id=venue_id))

    else:
      flash('An error occurred. Venue ' + name + ' could not be updated.')
      print("Error in edit_venue_submission()")
      return render_template('errors/500.html')

#  Create Venue
#  ----------------------------------------------------------------

@app.route('/venues/create', methods=['GET'])
def create_venue_form():
  form = VenueForm()
  return render_template('forms/new_venue.html', form=form)

@app.route('/venues/create', methods=['POST'])
def create_venue_submission():
  # DONE: insert form data as a new Venue record in the db, instead
  # DONE: modify data to be the data object returned from db insertion
  form = VenueForm()
  error = False

  name = form.name.data
  city = form.city.data
  state = form.state.data
  address = form.address.data
  phone = form.phone.data
  genres = form.genres.data
  web_site = form.web_site.data
  facebook_link = form.facebook_link.data
  image_link = form.image_link.data
  seeking_talent = True if form.seeking_talent.data == 'Yes' else False
  seeking_description = form.seeking_description.data

  try:
    venue = Venue(name=name, city=city, state=state, address=address,
    phone=phone, image_link=image_link, genres=genres, facebook_link=facebook_link,
    web_site=web_site, seeking_talent=seeking_talent, seeking_description=seeking_description)
    db.session.add(venue)
    db.session.commit()
  except:
    db.session.rollback()
    print(sys.exc_info())
    error = True
  finally:
    db.session.close()

  if error:
    flash('An error occurred. Venue ' + request.form['name'] + ' could not be listed.')
  else:
    flash('Venue ' + request.form['name'] + ' was successfully listed!')
  # DONE: on unsuccessful db insert, flash an error instead.
  # e.g., flash('An error occurred. Venue ' + data.name + ' could not be listed.')
  # see: http://flask.pocoo.org/docs/1.0/patterns/flashing/
  return render_template('pages/home.html')

@app.route('/venues/<venue_id>', methods=['DELETE'])
def delete_venue(venue_id):
  # TODO: Complete this endpoint for taking a venue_id, and using
  # SQLAlchemy ORM to delete a record. Handle cases where the session commit could fail.

  error = False
  try:
    venue = Venue.query.get(venue_id)
    db.session.delete(venue)
    db.session.commit()
  except:
    error = True
    db.session.rollback()
    print(sys.exc_info())
  finally:
    db.session.close()

  if error:
    flash('An error occurred. Venue ' + f'id = {venue_id}' + ' could not be deleted.')
  else:
    flash('Venue ' + request.form['name'] + ' was successfully listed!')

  # BONUS CHALLENGE: Implement a button to delete a Venue on a Venue Page, have it so that
  # clicking that button delete it from the db then redirect the user to the homepage
  return render_template('pages/home.html')


import math
from sys import argv
import multiprocessing as mp
import physics.pendulum as pendulum
import physics.gravity as gravity
import physics.pendulum_tension as pendulum_tension
import graph.integration_method as integration_method

def lock_distance(body: pendulum.Pendulum) -> None:
    """post-step function to ensure distance from pivot stays constant.

    post-step function to ensure distance from pivot stays constant
    by resetting the distance.

    Args:
        body (pendulum.Pendulum): The Pendulum Body to reset.
    """
    if (body.distance_from_pivot() - body.length < 0.01) or (abs(body.position.seek(-1)[0][1]) > 0.1):
        return

    angle = body.angle_from_pivot()
    body.position.pop()
    body.position.insert(body.time, math.cos(angle) * body.length,
                                    math.sin(angle) * body.length)

class ResetVelocity:
    """ResetVelocity represents a function that resets a Pendulum's velocity.

    ResetVelocity represents a function that sets a Pendulum's
    velocity to an expected velocity when it reaches its lowest point
    (also referred to as the base).

    """

    def __init__(self, base_velocity: float):
        """Initializes ResetVelocity with a base velocity.

        Initializes ResetVelocity with a base velocity.

        Args:
            base_velocity (float): The velocity of the pendulum at
            its base.
        """
        self.base_velocity = base_velocity

    def __call__(self, body: pendulum.Pendulum) -> None:
        """post-step function to ensure a Pendulum's velocity is correct.

        post-step function to ensure a Pendulum's velocity is correct
        when it is at its base (-90 degrees).

        Args:
            body (Pendulum): The Pendulum Body that is being reset.
        """
        if abs(body.angle_from_pivot() + math.pi/2) > 0.01: #
            return

        velocity = abs(body.velocity.x.seek(-1)[1])

        if self.base_velocity:
            time, v = body.velocity.x.pop()
            if v > 0:
                body.velocity.x.insert(time, self.base_velocity)
            else:
                body.velocity.x.insert(time, -self.base_velocity)
        else:
            self.base_velocity = velocity

def compile_simulation(degrees: int):
    """simulates a default Pendulum.

    Creates and simulates a default Pendulum (unless parameters are
    passed) with the Gravity and PendulumTension forces added.

    Args:
        angle (float, optional): The initial angle of the Pendulum.
        Defaults to 0.
        mass (float, optional): The mass of the Pendulum. Defaults
        to 1.
        dt (float, optional): The delta-time of the Pendulum.
        Defaults to 10**-3.
        length (float, optional): The length of the Pendulum.
        Defaults to 1.
        gravitational_source_mass (float, optional): The mass of the
        external actor. Defaults to 5.9722*10**24
        (mass of earth).
        gravitational_source_radius (float, optional): The radius of
        the external actor. Defaults to 6.371*10**6 (radius of earth).
        integration_method (IntegrationMethod, optional): The default
        integration method. Defaults to
        integration_method.IntegrationMethod.EULER.
    """
    #SIMULATION PARAMETERS
    angle = math.radians(-90 + degrees)
    mass = 0.1
    dt = 10 ** -3 * 1/2
    length = 0.1476
    gravitational_source_mass = 5.9722 * 10 ** 24 #Earth's mass
    gravitational_source_radius = 6.371 * 10 ** 6 #Earth's radius
    seconds = 3
    numerical_approximation = integration_method.IntegrationMethod.RK4

    #SAVE FILE'S PARAMETERS
    save_folder = argv[4]
    save_file_name = f'{save_folder}/{degrees} degrees.json'
    save_framerate = 60

    pendulum_obj = pendulum.Pendulum(
        mass = mass,
        dt = dt,
        pivot = [0, 0],
        length = length
    ).set_state(
        x_i = length * math.cos(angle),
        y_i = length * math.sin(angle),
        integration_method = numerical_approximation
    ).add_force(
        gravity.Gravity, gravitational_source_mass, gravitational_source_radius
    ).add_force(
        pendulum_tension.PendulumTension
    )

    for iterations in range(int((dt ** -1) * seconds)):
        if iterations % (dt ** -1) == 0:
            print(f'{int(iterations // (dt ** -1)) + 1}/{seconds} sec compiled for {degrees} degrees')
        pendulum_obj.step()

    pendulum_obj.save(save_file_name, framerate = save_framerate)

if __name__ == '__main__':
    #SIMULATE MULTIPLE ANGLES (RANGING FROM START TO STOP) SIMULTANEOUSLY
    start = int(argv[1])
    stop = int(argv[2])
    increment = int(argv[3])

    with mp.Pool() as pool:
        pool.map(compile_simulation, range(start, stop + increment, increment))
    print('Compilation finished')


import unittest

import torch

from models.matcher import HungarianMatcher
from models.position_encoding import PositionEmbeddingSine, PositionEmbeddingLearned
from models.backbone import Backbone, Joiner, BackboneBase
from util import box_ops
from util.misc import nested_tensor_from_tensor_list
from hubconf import detr_resnet50, detr_resnet50_panoptic

class Tester(unittest.TestCase):

    def test_box_cxcywh_to_xyxy(self):
        t = torch.rand(10, 4)
        r = box_ops.box_xyxy_to_cxcywh(box_ops.box_cxcywh_to_xyxy(t))
        self.assertLess((t - r).abs().max(), 1e-5)

    @staticmethod
    def indices_torch2python(indices):
        return [(i.tolist(), j.tolist()) for i, j in indices]

    def test_hungarian(self):
        n_queries, n_targets, n_classes = 100, 15, 91
        logits = torch.rand(1, n_queries, n_classes + 1)
        boxes = torch.rand(1, n_queries, 4)
        tgt_labels = torch.randint(high=n_classes, size=(n_targets,))
        tgt_boxes = torch.rand(n_targets, 4)
        matcher = HungarianMatcher()
        targets = [{'labels': tgt_labels, 'boxes': tgt_boxes}]
        indices_single = matcher({'pred_logits': logits, 'pred_boxes': boxes}, targets)
        indices_batched = matcher({'pred_logits': logits.repeat(2, 1, 1),
                                   'pred_boxes': boxes.repeat(2, 1, 1)}, targets * 2)
        self.assertEqual(len(indices_single[0][0]), n_targets)
        self.assertEqual(len(indices_single[0][1]), n_targets)
        self.assertEqual(self.indices_torch2python(indices_single),
                         self.indices_torch2python([indices_batched[0]]))
        self.assertEqual(self.indices_torch2python(indices_single),
                         self.indices_torch2python([indices_batched[1]]))

        # test with empty targets
        tgt_labels_empty = torch.randint(high=n_classes, size=(0,))
        tgt_boxes_empty = torch.rand(0, 4)
        targets_empty = [{'labels': tgt_labels_empty, 'boxes': tgt_boxes_empty}]
        indices = matcher({'pred_logits': logits.repeat(2, 1, 1),
                           'pred_boxes': boxes.repeat(2, 1, 1)}, targets + targets_empty)
        self.assertEqual(len(indices[1][0]), 0)
        indices = matcher({'pred_logits': logits.repeat(2, 1, 1),
                           'pred_boxes': boxes.repeat(2, 1, 1)}, targets_empty * 2)
        self.assertEqual(len(indices[0][0]), 0)

    def test_position_encoding_script(self):
        m1, m2 = PositionEmbeddingSine(), PositionEmbeddingLearned()
        mm1, mm2 = torch.jit.script(m1), torch.jit.script(m2)  # noqa

    def test_backbone_script(self):
        backbone = Backbone('resnet50', True, False, False)
        torch.jit.script(backbone)  # noqa

    def test_model_script_detection(self):
        model = detr_resnet50(pretrained=False).eval()
        scripted_model = torch.jit.script(model)
        x = nested_tensor_from_tensor_list([torch.rand(3, 200, 200), torch.rand(3, 200, 250)])
        out = model(x)
        out_script = scripted_model(x)
        self.assertTrue(out["pred_logits"].equal(out_script["pred_logits"]))
        self.assertTrue(out["pred_boxes"].equal(out_script["pred_boxes"]))

    def test_model_script_panoptic(self):
        model = detr_resnet50_panoptic(pretrained=False).eval()
        scripted_model = torch.jit.script(model)
        x = nested_tensor_from_tensor_list([torch.rand(3, 200, 200), torch.rand(3, 200, 250)])
        out = model(x)
        out_script = scripted_model(x)
        self.assertTrue(out["pred_logits"].equal(out_script["pred_logits"]))
        self.assertTrue(out["pred_boxes"].equal(out_script["pred_boxes"]))
        self.assertTrue(out["pred_masks"].equal(out_script["pred_masks"]))

if __name__ == '__main__':
    unittest.main()