import pytest
import numpy as np
import h5py
import tempfile
import pathlib
import hashlib
from itertools import product

import ophys_etl.utils.thumbnail_video_utils as thumbnail_utils
from ophys_etl.types import ExtractROI
from ophys_etl.utils.thumbnail_video_generator import (
    VideoGenerator)

@pytest.fixture(scope='session')
def list_of_roi():
    """
    A list of ExtractROIs
    """
    output = []
    rng = np.random.default_rng(11231)
    for ii in range(10):
        x0 = int(rng.integers(0, 30))
        y0 = int(rng.integers(0, 30))
        width = int(rng.integers(4, 10))
        height = int(rng.integers(4, 10))
        mask = rng.integers(0, 2, size=(height, width)).astype(bool)

        # because np.ints are not JSON serializable
        real_mask = []
        for row in mask:
            this_row = []
            for el in row:
                if el:
                    this_row.append(True)
                else:
                    this_row.append(False)
            real_mask.append(this_row)

        if ii % 2 == 0:
            valid_roi = True
        else:
            valid_roi = False
        roi = ExtractROI(x=x0, width=width,
                         y=y0, height=height,
                         valid=valid_roi,
                         mask=real_mask,
                         id=ii)
        output.append(roi)
    return output

@pytest.fixture(scope='session')
def example_video(tmpdir_factory):
    """
    Write an example video to an HDF5 file; return the
    path to that file.
    """
    tmpdir = pathlib.Path(tmpdir_factory.mktemp('eg_video'))
    rng = np.random.default_rng(121311)
    nt = 100
    nrows = 50
    ncols = 50
    data = rng.integers(0, 1000, (nt, nrows, ncols))
    fname = tempfile.mkstemp(dir=tmpdir,
                             prefix='video_generator_example_video_',
                             suffix='.h5')[1]
    with h5py.File(fname, 'w') as out_file:
        out_file.create_dataset('data', data=data)
    fname = pathlib.Path(fname)
    yield fname
    fname.unlink()
    tmpdir.rmdir()

@pytest.fixture
def example_roi():
    """an ExtractROI"""
    x = 11
    y = 7
    height = 20
    width = 35
    rng = np.random.default_rng(8123)
    mask = rng.integers(0, 2, (height, width)).astype(bool)
    roi = ExtractROI(x=x,
                     y=y,
                     height=height,
                     width=width,
                     mask=mask,
                     valid=True,
                     id=999)
    return roi

def compare_hashes(fname0, fname1):
    """
    Assert that two files have the same md5 checksum
    """
    hash0 = hashlib.md5()
    hash1 = hashlib.md5()
    with open(fname0, 'rb') as in_file:
        data = in_file.read()
        hash0.update(data)
    with open(fname1, 'rb') as in_file:
        data = in_file.read()
        hash1.update(data)
    assert hash0.hexdigest() == hash1.hexdigest()

def test_video_generator_exception():
    """test that VideoGenerator raises an exception when given
    a bogus file name"""
    with pytest.raises(RuntimeError, match='is not a file'):
        VideoGenerator('not_a_file.h5')

@pytest.mark.parametrize('origin, frame_shape, timesteps, as_array',
                         product((None, (5, 5)),
                                 (None, (16, 20)),
                                 (None, np.array([1, 4, 5, 17,
                                                  19, 23, 23, 25, 38])),
                                 (True, False)))
def test_get_thumbnail_video(tmpdir,
                             example_video,
                             origin,
                             frame_shape,
                             timesteps,
                             as_array):
    """
    Test that results of VideoGenerator and by-hand invocation of
    thumbnail_utils.thumbnail_video_from_path are identical
    """
    video_tmpdir = pathlib.Path(tmpdir) / 'video_temp'

    if as_array:
        with h5py.File(example_video, 'r') as in_file:
            generator = VideoGenerator(
                             video_data=in_file['data'][()],
                             tmp_dir=video_tmpdir)
    else:
        generator = VideoGenerator(video_path=example_video,
                                   tmp_dir=video_tmpdir)

    fps = 11
    quality = 6

    thumbnail = generator.get_thumbnail_video(origin=origin,
                                              frame_shape=frame_shape,
                                              timesteps=timesteps,
                                              fps=fps,
                                              quality=quality)

    assert thumbnail.video_path.is_file()

    if origin is None:
        origin = (0, 0)
    if frame_shape is None:
        with h5py.File(example_video, 'r') as in_file:
            frame_shape = in_file['data'].shape

    if as_array:
        with h5py.File(example_video, 'r') as in_file:
            expected = thumbnail_utils.thumbnail_video_from_array(
                            in_file['data'][()],
                            origin,
                            frame_shape,
                            timesteps=timesteps,
                            tmp_dir=video_tmpdir,
                            fps=fps,
                            quality=quality)
    else:
        expected = thumbnail_utils.thumbnail_video_from_path(
                        pathlib.Path(example_video),
                        origin,
                        frame_shape,
                        timesteps=timesteps,
                        tmp_dir=video_tmpdir,
                        fps=fps,
                        quality=quality,
                        min_max=generator.min_max)

    assert expected.video_path.is_file()
    assert not expected.video_path == thumbnail.video_path
    compare_hashes(expected.video_path, thumbnail.video_path)

    # to make sure we don't needlessly clutter CircleCIs
    # scratch space
    del thumbnail
    del expected
    del generator

@pytest.mark.parametrize('origin, frame_shape, timesteps, as_array',
                         product((None, (5, 5)),
                                 (None, (16, 20)),
                                 (None, np.array([1, 4, 5, 17,
                                                  19, 23, 23, 25, 38])),
                                 (True, False)))
def test_thumbnail_with_roi_list(
                             tmpdir,
                             example_video,
                             origin,
                             frame_shape,
                             timesteps,
                             list_of_roi,
                             as_array):
    """
    Just a smoketest that we can generate a by-hand thumbnail with ROIs
    displayed in it
    """
    video_tmpdir = pathlib.Path(tmpdir) / 'video_temp'

    if as_array:
        with h5py.File(example_video, 'r') as in_file:
            generator = VideoGenerator(
                            video_data=in_file['data'][()],
                            tmp_dir=video_tmpdir)
    else:
        generator = VideoGenerator(video_path=example_video,
                                   tmp_dir=video_tmpdir)

    fps = 11
    quality = 6

    thumbnail = generator.get_thumbnail_video(origin=origin,
                                              frame_shape=frame_shape,
                                              timesteps=timesteps,
                                              fps=fps,
                                              quality=quality,
                                              rois=list_of_roi)

    assert thumbnail.video_path.is_file()

    # to make sure we don't needlessly clutter CircleCIs
    # scratch space
    del thumbnail

    thumbnail = generator.get_thumbnail_video(origin=origin,
                                              frame_shape=frame_shape,
                                              timesteps=timesteps,
                                              fps=fps,
                                              quality=quality,
                                              rois=list_of_roi,
                                              valid_only=True)

    assert thumbnail.video_path.is_file()

    # to make sure we don't needlessly clutter CircleCIs
    # scratch space
    del thumbnail

    roi_lookup = {roi['id']: roi for roi in list_of_roi}

    thumbnail = generator.get_thumbnail_video(origin=origin,
                                              frame_shape=frame_shape,
                                              timesteps=timesteps,
                                              fps=fps,
                                              quality=quality,
                                              rois=roi_lookup)

    assert thumbnail.video_path.is_file()

    # to make sure we don't needlessly clutter CircleCIs
    # scratch space
    del thumbnail

    thumbnail = generator.get_thumbnail_video(origin=origin,
                                              frame_shape=frame_shape,
                                              timesteps=timesteps,
                                              fps=fps,
                                              quality=quality,
                                              rois=roi_lookup,
                                              valid_only=True)

    assert thumbnail.video_path.is_file()

    # to make sure we don't needlessly clutter CircleCIs
    # scratch space
    del thumbnail
    del generator

@pytest.mark.parametrize('roi_color, timesteps, padding, as_array',
                         product((None, (122, 201, 53)),
                                 (None, np.array([1, 4, 5, 17, 19,
                                                  23, 23, 25, 38])),
                                 (0, 5, 10),
                                 (True, False)))
def test_get_thumbnail_video_from_roi(
                             tmpdir,
                             example_video,
                             example_roi,
                             roi_color,
                             timesteps,
                             padding,
                             as_array):
    """
    Test that results of VideoGenerator and by-hand invocation of
    thumbnail_utils.thumbnail_video_from_ROI are identical
    """
    video_tmpdir = pathlib.Path(tmpdir) / 'video_temp'
    if as_array:
        with h5py.File(example_video, 'r') as in_file:
            video_data = in_file['data'][()]
        generator = VideoGenerator(
                             video_data=video_data,
                             tmp_dir=video_tmpdir)
    else:
        generator = VideoGenerator(video_path=example_video,
                                   tmp_dir=video_tmpdir)

    fps = 11
    quality = 6

    thumbnail = generator.get_thumbnail_video_from_roi(
                                  roi=example_roi,
                                  padding=padding,
                                  roi_color=roi_color,
                                  timesteps=timesteps,
                                  fps=fps,
                                  quality=quality)

    assert thumbnail.video_path.is_file()

    if as_array:
        expected = thumbnail_utils.thumbnail_video_from_ROI(
                       video_data,
                       example_roi,
                       padding=padding,
                       roi_color=roi_color,
                       timesteps=timesteps,
                       tmp_dir=video_tmpdir,
                       fps=fps,
                       quality=quality)

    else:
        expected = thumbnail_utils.thumbnail_video_from_ROI(
                       pathlib.Path(example_video),
                       example_roi,
                       padding=padding,
                       roi_color=roi_color,
                       timesteps=timesteps,
                       tmp_dir=video_tmpdir,
                       fps=fps,
                       quality=quality,
                       min_max=generator.min_max)

    assert expected.video_path.is_file()
    assert not expected.video_path == thumbnail.video_path
    compare_hashes(expected.video_path, thumbnail.video_path)

    # to make sure we don't needlessly clutter CircleCIs
    # scratch space
    del thumbnail
    del expected
    del generator


import numpy as np
import random
import open3d as o3d
import argparse
import os

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description='Fit SMPL body to vertices.')
    parser.add_argument('--path', type=str, default="D:/Human_data/4D_data/data/canonical_meshes/",
                        help='Path to the 3D skeleton that corresponds to the 3D scan.')
    parser.add_argument('--verbose', type=bool, default=True,
                        help='Flag to record intermediate results')
    parser.add_argument('--size', type=int, default=210,
                        help='nb of scans')

    args = parser.parse_args()
    path = args.path

    for idx in range(0,args.size):
        #scan_path = path + "geometry/4dviews_nobi_retopo_" + str(idx).zfill(4) + ".obj"
        if not os.path.isfile(os.path.join(path , "PC_" + str(idx).zfill(4) + ".ply")):
            print("not find : " , os.path.join(path , "PC_" + str(idx).zfill(4) + ".ply"))
            continue

        pcd = o3d.io.read_point_cloud(os.path.join(path , "PC_" + str(idx).zfill(4) + ".ply"))
        print(pcd)

        print('run Poisson surface reconstruction')
        with o3d.utility.VerbosityContextManager(
                o3d.utility.VerbosityLevel.Debug) as cm:
            mesh, densities = o3d.geometry.TriangleMesh.create_from_point_cloud_poisson(
                pcd, depth=9)
        #vertices_to_remove = densities < np.quantile(densities, 0.01)
        #mesh.remove_vertices_by_mask(vertices_to_remove)
        print(mesh)
        o3d.io.write_triangle_mesh(os.path.join(path , "mesh_" + str(idx).zfill(4) + ".ply"), mesh, write_ascii=True)


"""
source: own
author: https://github.com/MarkShawn2020
create: Nov 08, 2022, 14:56
"""

from argparse import ArgumentParser

from src.pdf2images import pdf2images

from src.parseLZY.parseQuestionPage import splitProblems

if __name__ == '__main__':
    parser = ArgumentParser()
    subparsers = parser.add_subparsers(dest='command')

    parser_pdf2images = subparsers.add_parser('pdf2images')
    parser_pdf2images.add_argument('fp', help='pdf文档，本项目目前使用的是2021年李正元复习全书数学一习题册')
    parser_pdf2images.set_defaults(func=lambda x: pdf2images(fp=x.fp))

    parser_split_problems = subparsers.add_parser('split-problems')
    parser_split_problems.add_argument('fp', help='李正元习题册某一页的图片文件地址，在 data/images 下')
    parser_split_problems.set_defaults(func=lambda x: splitProblems(fp=x.fp))

    args = parser.parse_args()
    args.func(args)


from functools import partial
from base.models import Book
from base.serializers import BookSerializer
from rest_framework.response import Response
from rest_framework.decorators import api_view,permission_classes
from rest_framework.permissions import IsAuthenticated, IsAdminUser
from rest_framework.decorators import api_view
from rest_framework import status

# Create your views here.

@api_view(['GET'])
@permission_classes([IsAuthenticated])
def getBooks(request, pk=None):
    if pk is not None:
        book = Book.objects.get(id=pk)
        serializer = BookSerializer(book, many=False)
    else:
        books = Book.objects.all()
        serializer = BookSerializer(books, many=True)
    return Response(serializer.data)

@api_view(['POST'])
@permission_classes([IsAdminUser])
def createBook(request):
    data = request.data
    book = Book.objects.create(
        title = data['title'],
        author = data['author'],
        quantity = data['quantity']
    )
    serializer = BookSerializer(book, many=False)
    return Response(serializer.data)

@api_view(['PUT'])
@permission_classes([IsAdminUser])
def updateBook(request, pk):
    data= request.data
    book = Book.objects.get(id=pk)
    book.title = data['title']
    book.author = data['author']
    book.quantity = data['quantity']
    book.save()
    serializer = BookSerializer(book, many=False, partial=True)
    return Response(serializer.data)

@api_view(['DELETE'])
@permission_classes([IsAdminUser])
def deleteBook(request, pk):
    book = Book.objects.get(id=pk)
    book.delete()
    return Response({'message':'Book Deleted Successfully'})


from django.contrib import admin

from services.models import services
# Register your models here.

class servicesModelAdmin (admin.ModelAdmin):
	list_display = ["__unicode__", "last_updated_time"]
	class Meta:
		model = services

admin.site.register(services, servicesModelAdmin)


import pandas as pd
from PIL import Image, ImageDraw, ImageFont

data = pd.read_excel (r'C:\Users\Rahul sinha\Desktop\Python Learning Jupyter\gen.xlsx')
name_list = data["Name"].tolist()
for i in name_list:
    im = Image.open(r'C:\Users\Rahul sinha\Desktop\Python Learning Jupyter\new.jpg')
    d = ImageDraw.Draw(im)
    location = (100, 398)
    text_color = (0, 137, 209)
    font = ImageFont.truetype("arial.ttf", 120)
    d.text(location, i, fill = text_color, font = font)
    im.save("certificate_" + i + ".pdf")


# coding: utf-8

# In[4]:

import numpy as np
import pandas as pd
from datetime import datetime
from pandas_datareader import data as web
import matplotlib.pyplot as plt
import scipy.optimize as sco

# Dow Jones Industrial Average Tickers

# DJIA = ['JNJ','UTX','IBM','CVX','PG','KO','MCD','MSFT','DIS','VZ','NKE','AAPL','INTC', '^GSPC']
SETH = ['BIV','BLV','BND','VCIT','VFIAX','VYM','VO','VB','VWO','VSS','VGTSX','VNQ','PARWX']

DJIA = ['JNJ','IBM','CVX','PG','KO','MCD','MSFT','DIS','BA','NKE','AAPL','INTC','FIZZ','^GSPC']

# Dates
start = datetime(2010, 1, 1)
end = datetime.today()
# Grab data, change to weekly returns and write to CSV

# In[5]:

def getdata(listofstocks, startdate, enddate):
    for i in range(2):
        try:
            x = web.DataReader(listofstocks,"yahoo", startdate, enddate)['Adj Close']
            break
        except Exception as error:
            print("Could not pull the data see below error /n{}".format(error))
    df = pd.DataFrame(x)
    df = df.resample('W-FRI').last().sort_index(ascending=False) #changing data to weekly
    for row in range(len(df)-1):
        df.iloc[row] = df.iloc[row].div(df.iloc[row+1]) #return
    df = df.iloc[:-1]
    df = np.log(df)  #taking log return
    df.to_csv('SethFundData.csv', encoding='utf-8') #write to CSV
    return(df)

data = getdata(DJIA, start, end)
print((data).head(5))
print((data).tail(5))
# print(data[['AAPL']])

# In[6]:

mult = 52.1429 #number of weeks per year
numberofstocks = 13
# data = pd.read_csv('SethFundData.csv', index_col=False)

def split_data(dataframe, numberofstocks):
# 	numberofstocks = numberofstocks + 1
	temp1 = data.iloc[:,0:(numberofstocks)]
	temp2 = data[['^GSPC']]
	temp2.columns = ['S&P']
	df = pd.concat([temp1,temp2],axis = 1) # select number of columns
	sample_df = df.iloc[0:int(len(df.index)/5),:-1]
	trn_df = df.iloc[int(len(df.index)/5):len(df.index),:-1]
	trn_df_graph = df.iloc[int(len(df.index)/5):len(df.index),:]
	sample_bench = df.iloc[0:int(len(df.index)/5),-1]
	return(trn_df,sample_df,trn_df_graph, sample_bench)

trn_df = split_data(data,numberofstocks)[0]
trn_df_graph = split_data(data,numberofstocks)[2]
print((trn_df_graph).head(5))

# df = pd.concat([dataframe.iloc[:,0:(numberofstocks)],datafreme[['^GSPC']]], axis = 1)

# In[7]:

trn_df.mean() * mult

# In[8]:

trn_df.std() * mult

# In[9]:

trn_df.cov() * mult

# In[10]:

def graph_ret(dataframe, location):
	trn_ret_df = dataframe.sort_index(ascending=True)
	trn_ret_df.iloc[:,:] = trn_ret_df.iloc[:,:].add(1)
	for row in range(0,len(trn_ret_df.index), +1):
		trn_ret_df.iloc[row,:] = (trn_ret_df.iloc[row,:]).mul(trn_ret_df.iloc[row-1,:])
# 	trn_ret_df.set_index('Date',inplace=True)
	trn_ret_df.plot(figsize=(14,6))
	plt.savefig(location)
	plt.show()
	plt.close()
	return(trn_ret_df)
print(graph_ret(trn_df_graph, "./return.png"))

# In[11]:

noc=len(trn_df.columns) # total number for index of columns
print("Number of Columns={}".format(noc))

# In[12]:

def basic_ret(dataframe, noc):# annualized mean return
	np.random.seed(0)
	weights = np.random.random(noc) # initial random weights
	weights /= np.sum(weights) #take weight devide by sum of eights and save

	exp_ret = np.dot(dataframe.mean(), weights) * mult #annualized expected return with initial weights
	exp_var = np.dot(weights.T, np.dot(dataframe.cov() * mult, weights))
	exp_vol = np.sqrt(exp_var)
	return(exp_var, exp_vol, exp_ret)

# In[13]:

def simulation(dataframe, steps, noc):
	prets = []
	pvols = []
	for i in range(steps):
		np.random.seed(i)
		weights = np.random.random(noc)
		weights /= np.sum(weights)
		exp_ret = np.dot(dataframe.mean(), weights) * mult
		prets.append(exp_ret)
		exp_var = np.dot(weights.T, np.dot(dataframe.cov() * mult, weights))
		pvols.append(np.sqrt(exp_var))
	prets = np.array(prets)
	pvols = np.array(pvols)
	return(prets, pvols)

# In[14]:

prets, pvols = simulation(trn_df,10000,noc)[0:2]
print("Expected Max Sharp = {}".format(max(prets/pvols)))
exp_vol, exp_ret = basic_ret(trn_df, noc)[1:3]
plt.figure(figsize=(8,4))
plt.scatter(pvols,prets, c= prets / pvols, marker ="o")
plt.plot(exp_vol,exp_ret, 'r*', markersize=15.0)
plt.grid(True)
plt.xlabel("Expected Volatility: Risk")
plt.ylabel("Expected Return")
plt.colorbar(label = "Sharpe ratio")
plt.savefig("./scatterplot.png")
plt.show()
plt.close()

# In[15]:

def statistics(weights):
	weights = np.array(weights)
	pret = np.dot(trn_df.mean(), weights) * mult
	pvol = np.sqrt(np.dot(weights.T, np.dot(trn_df.cov() * mult, weights)))
	return(np.array([pret,pvol, pret / pvol]))

# In[16]:

base_weights = noc * [1/noc,]
print("baseweights = {}".format(np.array(base_weights).round(4)))

# In[17]:

cons = ({'type':'eq', 'fun':lambda x: np.sum(x) - 1}) #defining constraints
bnds = tuple((0, 1) for x in range(noc))
# stocknames = np.delete(np.array(trn_df.columns),(0), axis=0)
stocknames = np.array(trn_df.columns)
print(cons)
print(bnds)
print(stocknames)

# In[18]:

def min_sharpe(weights):
	return(-statistics(weights)[2])

# In[19]:

opts = sco.minimize(min_sharpe, base_weights, method='SLSQP', bounds=bnds, constraints=cons)
max_sharp_weights = np.column_stack((stocknames, opts['x'].T.round(3)))
max_sharp_weights = np.flipud(max_sharp_weights[max_sharp_weights[:,1].argsort()])
print("Max Sharp weights")
print(max_sharp_weights)
statoutnames = np.array(['return','volatility', 'sharp'])
pd.DataFrame(np.column_stack((statoutnames, statistics(opts['x']).T.round(4))).T)

# In[20]:

# prets, pvols = simulation(trn_df,2500,noc)[0:2]
exp_ret_sharp, exp_vol_sharp = statistics(opts['x'])[0:2]
print(exp_ret_sharp, exp_vol_sharp)
plt.figure(figsize=(12,6))
plt.scatter(pvols,prets, c = prets / pvols, marker ="o")
plt.plot(exp_vol_sharp,exp_ret_sharp, 'r*', markersize=15.0)
plt.grid(True)
plt.xlabel("Expected Volatility: Risk")
plt.ylabel("Expected Return")
plt.colorbar(label = "Sharpe ratio")
plt.savefig("./scatterplot.png")
plt.show()
plt.close()

# In[21]:

def max_ret(weights):
	return(-statistics(weights)[0])

# In[22]:

optr = sco.minimize(max_ret, base_weights, method='SLSQP', bounds=bnds, constraints=cons)
max_ret_weights = np.column_stack((stocknames, optr['x'].T.round(3)))
max_ret_weights = np.flipud(max_ret_weights[max_ret_weights[:,1].argsort()])
print("Maximize return weights")
print(max_ret_weights)
pd.DataFrame(np.column_stack((statoutnames, statistics(optr['x']).T.round(4))).T)

# In[23]:

# prets, pvols = simulation(trn_df,2500,noc)[0:2]
exp_ret_ret, exp_vol_ret = statistics(optr['x'])[0:2]
print(exp_ret_ret, exp_vol_ret)
plt.figure(figsize=(12,6))
plt.scatter(pvols,prets, c = prets / pvols, marker ="o")
plt.plot(exp_vol_ret,exp_ret_ret, 'r*', markersize=15.0)
plt.grid(True)
plt.xlabel("Expected Volatility: Risk")
plt.ylabel("Expected Return")
plt.colorbar(label = "Sharpe ratio")
plt.savefig("./scatterplot.png")
plt.show()
plt.close()

# In[24]:

def min_pvol(weights):   #New function to minimize
	return(statistics(weights)[1])

# In[25]:

optv = sco.minimize(min_pvol, base_weights, method='SLSQP', bounds=bnds, constraints=cons)
min_vol_weights = np.column_stack((stocknames, optv['x'].T.round(3)))
min_vol_weights = np.flipud(min_vol_weights[min_vol_weights[:,1].argsort()])
print("Minimize risk weights")
print(min_vol_weights)
pd.DataFrame(np.column_stack((statoutnames, statistics(optv['x']).T.round(4))).T)

# In[26]:

# prets, pvols = simulation(trn_df,2500,noc)[0:2]
exp_ret_vol, exp_vol_vol = statistics(optv['x'])[0:2]
print(exp_ret_vol, exp_vol_vol)
plt.figure(figsize=(12,6))
plt.scatter(pvols,prets, c = prets / pvols, marker ="o")
plt.plot(exp_vol_vol,exp_ret_vol, 'r*', markersize=15.0)
plt.grid(True)
plt.xlabel("Expected Volatility: Risk")
plt.ylabel("Expected Return")
plt.colorbar(label = "Sharpe ratio")
plt.savefig("./scatterplot.png")
plt.show()
plt.close()

# In[28]:

trets = np.linspace(0.10,0.18,14)
tvols = []
tweights = []

for tret in trets:
	cons = ({'type':'eq', 'fun':lambda x: statistics(x)[0] - tret},   #New constraints
			{'type':'eq', 'fun':lambda x: np.sum(x) - 1})
	res = sco.minimize(min_pvol, base_weights, method='SLSQP', bounds=bnds, constraints=cons)
	tvols.append(res['fun'])
	# temp2 = np.column_stack((res['fun'], res['x'].round(3)))
	# temp1 = np.column_stack(tret)

	# tweights.append(temp2)
	# # tweights.append(tret)
	# tweights.append(res['fun'])
	tweights.append(res['x'].round(3))
tvols = np.array(tvols)
tweights = np.array(tweights)

front_line = np.column_stack((trets, tvols)).round(3)
tweights = np.column_stack((front_line,tweights))
tweights = np.transpose(tweights)

dummy = ['return','volatility']
dummy = np.array(dummy)
dummy = np.append(dummy, stocknames)
dummy = np.column_stack((dummy,tweights))
pd.DataFrame(dummy)

# In[30]:

plt.figure(figsize=(8,4))
plt.scatter(pvols,prets, c=prets / pvols, marker ="o")
plt.scatter(tvols,trets, c=trets / tvols, marker ="x")
plt.plot(statistics(opts['x'])[1],statistics(opts['x'])[0], 'r*', markersize=15.0)
plt.plot(statistics(optv['x'])[1],statistics(optv['x'])[0], 'y*', markersize=15.0)
# plt.plot(statistics(optr['x'])[1],statistics(optr['x'])[0], 'g*', markersize=15.0)
plt.grid(True)
plt.xlabel("Expected Volatility: Risk")
plt.ylabel("Expected Return")
plt.colorbar(label = "Sharpe ratio")
plt.savefig("./FrontierLine.png")
plt.show()
plt.close()

# In[32]:

sample_df = split_data(data, numberofstocks)[1]
sample_df.head(10)

# In[33]:

sample_df.mean() * mult

# In[34]:

sample_df.std() * mult

# In[35]:

##Choose your column
xx = 11
optweights = tweights[:,xx - 1][2:]

# In[36]:

def test(weights, dataframe):
	weights = np.array(weights)
	pret = np.dot(dataframe.mean(), weights) * mult
	pvol = np.sqrt(np.dot(weights.T, np.dot(dataframe.cov() * mult, weights)))
	return(np.array([pret,pvol, pret / pvol]))

# In[37]:

pd.DataFrame(np.column_stack((statoutnames, test(optweights,sample_df).T.round(4))).T)

# In[38]:

sample_banch = trn_df_graph = split_data(data,numberofstocks)[3]
sample_banch.mean() * mult
teststats = np.array([sample_banch.mean() * mult, sample_banch.std() * mult, ((sample_banch.mean() * mult) / (sample_banch.std() * mult))])
pd.DataFrame(np.column_stack((statoutnames, teststats.T.round(5))).T)

# In[39]:

def max_test(weights,dataframe):
	return(test(weights,dataframe)[1])

test_cons = ({'type':'eq', 'fun':lambda x: np.sum(x) - 1}) #defining constraints

# In[40]:

opt_test = sco.minimize(max_test, base_weights,args=(sample_df), method='SLSQP', bounds=bnds, constraints=test_cons)
min_vol_test = np.column_stack((stocknames, opt_test['x'].T.round(3)))
min_vol_test = np.flipud(min_vol_test[min_vol_test[:,1].argsort()])
print("Minimize risk weights")
print(min_vol_test)
# pd.DataFrame(np.column_stack((statoutnames, test(opt_test['x'],).T.round(4))).T)


from django.urls import path, include
from rest_framework_simplejwt.views import TokenVerifyView, TokenRefreshView
from . import views

urlpatterns = [
    path("api/token/verify/", TokenVerifyView.as_view(), name="token_verify"),
    path("refresh", TokenRefreshView.as_view(), name="token_refresh"),
    path("login/<str:social_type>", views.SocialLoginView.as_view(), name="social_login"),
    path("mypage", views.MypageView.as_view(), name="mypage"),
    path("introduction", views.UserIntroductionView.as_view(), name="introduction"),
]


import pygame

health_box_image = pygame.image.load('images/icons/health_box.png')
knife_box_image = pygame.image.load('images/icons/knife_box.png')
item_boxes = {
    'Health' : health_box_image,
    'Knives' : knife_box_image
}

class Knives(pygame.sprite.Sprite):
    def __init__(self, x, y, direction):
        super().__init__()
        self.speed = 10
        self.image = pygame.image.load('images/icons/knife.png').convert_alpha()
        self.rect = self.image.get_rect()
        self.rect.center = (x, y)
        self.direction = direction

    def update(self, screen_width, enemy_group, knife_group, obstacle_list, screen_scroll):
        # move the knife
        self.rect.x += self.direction * self.speed + screen_scroll
        # check if knife has gone of screen
        if self.rect.right < 0 or self.rect.left > screen_width:
            self.kill()

        # check for collision with level
        for tile in obstacle_list:
            if tile [1].colliderect(self.rect):
                self.kill()

        for enemy in enemy_group:
            if pygame.sprite.spritecollide(enemy, knife_group, False):
                if enemy.alive:
                    enemy.health -= 50
                    self.kill()

class Water(pygame.sprite.Sprite):
    def __init__(self, img, x, y, tile_size):
        super().__init__()
        self.image = img
        self.rect = self.image.get_rect()
        self.rect.midtop = (x + tile_size // 2, y + (tile_size - self.image.get_height()))

    def update(self, screen_scroll):
        self.rect.x += screen_scroll

class Exit(pygame.sprite.Sprite):
    def __init__(self, img, x, y, tile_size):
        super().__init__()
        self.image = img
        self.rect = self.image.get_rect()
        self.rect.midtop = (x + tile_size // 2, y + (tile_size - self.image.get_height()))

    def update(self, screen_scroll):
        self.rect.x += screen_scroll

class Decoration(pygame.sprite.Sprite):
    def __init__(self, img, x, y, tile_size):
        super().__init__()
        self.image = img
        self.rect = self.image.get_rect()
        self.rect.midtop = (x + tile_size // 2, y + (tile_size - self.image.get_height()))

    def update(self, screen_scroll):
        self.rect.x += screen_scroll

class ItemBox(pygame.sprite.Sprite):
    def __init__(self, item_type, x, y, tile_size):
        super().__init__()
        self.item_type = item_type
        self.image = item_boxes[self.item_type]
        self.rect = self.image.get_rect()
        self.rect.midtop = (x + tile_size // 2, y + (tile_size - self.image.get_height()))

    def update(self, knight, screen_scroll):
        # scroll
        self.rect.x += screen_scroll
        if pygame.sprite.collide_rect(self, knight):
            # check the type of box
            if self.item_type == 'Health':
                knight.health += 25
                if knight.health > knight.max_health:
                    knight.health = knight.max_health
            elif self.item_type == 'Knives':
                knight.knives += 5
            # delete the box
            self.kill()


import cocotb
from logging import getLogger
from cocotb.monitors import BusMonitor
from cocotb.triggers import ReadOnly
from cocotb.result import TestFailure

class BaseMonitor(BusMonitor):

	def __init__(self, entity, clock, scoreboard):
		BusMonitor.__init__(self, entity, "", clock)

		self._expected = None
		self._log = getLogger(scoreboard.log.name + '.' + self.name)
		self._scoreboard = scoreboard
		scoreboard.add_interface(self, [], compare_fn=self.compare)

	def _print_expected(self, key, value):
		try:
			self._log.info("    %s: %s (0x%x)",
			               key, str(value), int(value))
		except :
			self._log.info("    %s: %s", key, str(value))

	def _print_diff(self, key, value, expect):
		try:
			self._log.info("    %s: %s != %s (0x%x != 0x%x)",
			               key, str(expect), str(value),
			               int(expect), int(value))
		except :
			self._log.info("    %s: %s != %s",
			               key, str(expect), str(value))

	def failure(self, message):
		self._scoreboard.errors += 1
		if self._scoreboard._imm:
			raise TestFailure(message)
		else:
			self._log.error(message)

	def compare(self, transaction):
		if self._expected == None:
			return

		self._log.debug("Checking " + self._expected["name"] + "...")

		wrong = False
		# validate transaction against signals present into expected
		# output
		for k,v in transaction.items():
			if (self._expected.has_key(k) and
			    (k != "name") and
			    (not str(v) == str(self._expected[k]))):
				wrong = True
				break

		if wrong:
			self._log.error("Received unexpected %s" %
			                (self._expected.has_key("name") == True
			                 and self._expected["name"] or
			                 "anonymous transaction"))

			self._log.info("Expected:")
			for k, v in sorted(self._expected.items()):
				if k != "name":
					self._print_expected(k, v)

			self._log.info("Received:")
			for k, v in sorted(transaction.items()):
				if k != "name":
					self._print_expected(k, v)

			self._log.info("Diff:")
			for k, v in sorted(transaction.items()):
				if ((k == "name") or
				    (not self._expected.has_key(k))):
					continue
				if not str(v) == str(self._expected[k]):
					self._print_diff(k, v,
					                 self._expected[k])

			self._scoreboard.errors += 1
			if self._scoreboard._imm:
				raise TestFailure(("Received unexpected ",
				                   "transaction"))

		self._expected = None

	@cocotb.coroutine
	def expect(self, expected):
		assert(self._expected == None)
		self._expected = expected

		yield self.wait_for_recv()

	@cocotb.coroutine
	def _monitor_recv(self):
		while True:
			yield ReadOnly()

			# build transaction from the entire list of declared
			# signals
			transaction = {}
			for sig in self._signals:
				transaction[sig] = getattr(self.bus, sig)

			self._recv(transaction)


from rest_framework import routers, serializers, viewsets
from .models import Flight

class FlightSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = Flight
        fields = [
            'id',
            'flight_kind',
            'flight_date',
            'flight_co',
            'flight_number',
            'flight_ac_type',
            'flight_ac_reg_number',
        ]


import click
import os

import numpy as np

from achilles.experiment import TestTube
from achilles.model import AchillesModel
from pathlib import Path

@click.command()
@click.option(
    "--training_dir",
    "-t",
    metavar="",
    default=None,
    required=True,
    show_default=True,
    help="Training directory from which to parse models.",
)
@click.option(
    "--evaluation_dir",
    "-e",
    metavar="",
    default=None,
    required=True,
    show_default=True,
    help="Evaluation HD5 file from AchillesModel.",
)
@click.option(
    "--outdir",
    "-o",
    metavar="",
    default=None,
    required=True,
    show_default=True,
    help="Output directory.",
)
def lab(outdir, training_dir, evaluation_dir):

    """ Run predictions in a test tube, not completed. """

    tube = TestTube(outdir=outdir)

    tube.run_predictions(
        training_dir=training_dir,
        evaluation_path=evaluation_dir,
        prefix="lab",
        mode="pairwise",
        batch_size=800,
    )


from django.core.management.base import BaseCommand
from django.contrib.auth.models import User
from oss.models import Site, Installation, Telescope
from oss.models import Instrument, InstrumentCapabilities
from oss.management.commands import lco, ingest_utils

class Command(BaseCommand):
    args = ''
    help = ''

    def _ingest_telescopes(self):
        instrument_list = lco.fetch_lco_instruments()

        for instrument in instrument_list:
            qs = Telescope.objects.filter(tel_code=instrument['tel_code'])
            (tel, message) = ingest_utils.test_qs_unique_result(qs, [instrument['tel_code']])

            if message == 'OK':
                (new_instrument, stat) = Instrument.objects.get_or_create(name=instrument['name'],
                                                                  telescope=tel,
                                                                  wavelength=instrument['wavelength'],
                                                                  url=instrument['url'])

                for capability in instrument['capabilities']:
                    qs = InstrumentCapabilities.objects.filter(descriptor=capability)
                    if len(qs) == 0:
                        cap = InstrumentCapabilities.objects.create(descriptor=capability)
                    else:
                        cap = qs[0]

                    new_instrument.capabilities.add(cap)
                    new_instrument.save()

                print('Ingested instrument '+instrument['name']+' at '+instrument['tel_code'])

            else:
                print(message)

    def handle(self,*args, **options):
        self._ingest_telescopes()


from django.conf import settings
from django.conf.urls.static import static
from django.contrib import admin
from django.urls import path, include

urlpatterns = [
    path('jet/', include('jet.urls', 'jet')),  # Django JET URLS
    path('admin/', admin.site.urls),
    path('', include('app.urls')),
    path('ckeditor/', include('ckeditor_uploader.urls')),
    path('froala_editor/', include('froala_editor.urls')),
    path('admin/doc/', include('django.contrib.admindocs.urls'))
]
if settings.DEBUG:
    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


import streamlit as st
import pandas as pd
import numpy as np
import pickle
import xgboost
import sklearn

file1 = open('n2opredictor_new1.pkl', 'rb')
xgb = pickle.load(file1)
file1.close()

data = pd.read_csv("new_trained_data.csv")
data = data.iloc[:,1:]
#------------------------------------------------------------

st.title("N2O Predictor")

month = st.selectbox('Month', ['January','February','March','April','May','June','July','August','September','October','November','December'])

year = st.selectbox('Year',[2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021,202,2023,2024,2025,2026,2027])

pp_2 = st.number_input('PP2')

pp_7 = st.number_input('PP7')

airt = st.number_input('AIRT')

daftd = st.number_input('DAF_TD')

dafsd = st.number_input('DAF_SD')

wfp = st.number_input('WFPS25cm')

nh4 = st.number_input('NH4 Content')

no3 = st.number_input('NO3 Content')

#------------------------------------------------------------------------------------------------------------

if st.button('Predict NO2 Flux'):

    months = {'January':1,'February':2,'March':3,'April':4,'May':5,'June':6,'July':7,'August':8,'September':9,'October':10,'November':11,'December':12}

    mnth = months[month]

    query = np.array([[airt, pp_2, nh4, pp_7, wfp, year, no3, daftd, dafsd,mnth]])

    prediction = xgb.predict(query)

    st.title("Predicted NO2 Flux is " +
             str(prediction))


"""Assets module."""
import os
import logging
import pygame
import pyscroll
import pytmx

from .locals import *  # noqa
from .utils import ContainerAware

logger = logging.getLogger(__name__)

class AssetManager(ContainerAware):

    """Asset manager."""

    def __init__(self):
        """Constructor."""
        logger.debug('Initializing AssetManager')

        self.configuration = self.container.get(Configuration)
        self.base_path = self.configuration.get('akurra.assets.base_path', 'assets')

    def get_path(self, asset_path):
        """
        Return a path to an asset while taking distributions and base paths into account.

        :param asset_path: Relative path of asset to process.

        """
        return os.path.join(self.base_path, asset_path)

    def get_sound(self, asset_path):
        """
        Return an sfx object (OGG only for now).

        :param asset_path: Relative path of asset to process.

        """
        path = self.get_path(asset_path)
        sound = pygame.mixer.Sound(path)

        return sound

    def get_image(self, asset_path, colorkey=None, alpha=False):
        """
        Return an image by processing an asset.

        :param asset_path: Relative path of asset to process.

        """
        path = self.get_path(asset_path)
        image = pygame.image.load(path)
        image = image.convert_alpha() if alpha else image.convert()

        if colorkey:
            image.set_colorkey(colorkey)

        return image

    def get_tmx_data(self, asset_path):
        """
        Return TMX data by processing an asset.

        :param asset_path: Relative path of asset to process.

        """
        path = self.get_path(asset_path)
        tmx_data = pytmx.load_pygame(path)

        return tmx_data

    def get_map_data(self, asset_path):
        """
        Return map data by processing an asset.

        :param asset_path: Relative path of asset to process.

        """
        tmx_data = self.get_tmx_data(asset_path)
        map_data = pyscroll.data.TiledMapData(tmx_data)

        return map_data


from transformers import  AutoTokenizer, AutoModelForSequenceClassification, pipeline
#import torch.nn.functional as F
from flask import request, Flask
import numpy as np
#import torch
import json
import re

app = Flask(__name__)

def get_pipleine():
    tokenizer = AutoTokenizer.from_pretrained("SarmadBashir/Issue_Assignment", model_max_length = 128)
    model = AutoModelForSequenceClassification.from_pretrained("SarmadBashir/Issue_Assignment")
    pipe = pipeline("text-classification", model=model, tokenizer=tokenizer)
    return pipe

pipe = get_pipleine()

#path = '../issue_assigner/model'
#tokenizer = BertTokenizerFast.from_pretrained(path, lower_case=True)
#model = BertForSequenceClassification.from_pretrained(path)

def preprocess(text):
    line = text.lower()
    line = re.sub(r"http\S+", "", line)
    line = re.sub("[^A-Za-z]+", " ", line)
    line = re.sub('\s+', ' ', line)
    line = line.replace('\t',' ')
    line = line.replace('\n',' ')
    line = line.replace('\r',' ')
    line = line.replace(',','')
    line = line.replace('-',' ')
    line = ' '.join(line.split())

    return line

#def get_prediction(input_text, tokenizer, model):
def get_prediction(input_text, pipe):
    map_labels = {
                 'LABEL_0': 'bug',
                 'LABEL_1': 'enhancement',
                 'LABEL_2': 'internal improvement',
                 'LABEL_3': 'hilla',
                 'LABEL_4': 'a11y',
                 'LABEL_5': 'documentation'
             }

    output = pipe(input_text)

    label = map_labels.get(output[0]['label'])
    score = round(output[0]['score'], 2)

    #MAX_LENGTH =  128
    #test_encodings = tokenizer(input_text, truncation=True,
    #                           padding=True,
    #                           max_length=MAX_LENGTH,
    #                           return_tensors="pt")

    #with torch.no_grad():
    #    logits = model(**test_encodings).logits

    #predictions = np.argmax(logits, axis=1)
    #top_label_int =  predictions.tolist()[0]
    #top_label = map_labels.get(top_label_int)

    #probabilities = F.softmax(logits, dim=1)
    #top_probability = probabilities[0].tolist()[top_label_int]
    #top_probability = round(top_probability, 2)

    return label, score

@app.route("/assigner", methods=['GET'])
def run():

    #issue_text = request.args.get('issue_text', '')
    data = request.get_json()
    issue_text = data['text']
    print(issue_text)
    if issue_text == '':
        return app.response_class(response='Provide valid input text',
                                  status=400,
                                  mimetype='application/json'
                                  )

    input_text =  preprocess(issue_text)
    top_label, top_probability = get_prediction(input_text,
                                                pipe)
    #                                            tokenizer,
    #                                            model)

    result = {'prediction': {'label': top_label,
                             'probability': top_probability
                             }
              }

    response = app.response_class(response=json.dumps(result),
                                  status=200,
                                  mimetype='application/json'
                                  )
    return response

if __name__ == "__main__":
    app.run(debug=True, use_reloader=False, host='0.0.0.0', port=5000)


# coding=utf-8
# python3

import os
from openpyxl import Workbook
from time import ctime
# from korean import Noun

count = colus = cnt = 1
workbook = Workbook()
worksheet = workbook.worksheets[0]

def save2xlsx(fullname, col):
    global count, colus, cnt
    index = count if col == 1 else colus if col == 2 else cnt
    worksheet.cell(row=index, column=col).value = fullname
    if(col == 1):
        count += 1
    elif(col == 2):
        colus += 1
    else:
        cnt += 1

def removeLyric(fullname):
    print(fullname)
    if '.lrc' in fullname.lower():
        try:
            os.remove(fullname)
            save2xlsx(fullname, 3)
        except:
            pass

def findMusics(fullname, col):
    if '.ncm' in fullname.lower():
        save2xlsx(fullname, col)
    if '???' in fullname:
        removeLyric(fullname)

def findImages(fullname, col):
    if '.nef' in fullname.lower():
        save2xlsx(fullname, col)

def searchFolder(dir, col):
    for file in os.listdir(dir):
        fullname = dir + '\\' + file
        # if '2018VN' in fullname:
        if os.path.isdir(fullname):
            searchFolder(fullname, col)
        else:
            # findMusics(fullname, col)
            findImages(fullname, col)

if __name__ == "__main__":
    print('\n>>> start ' + ctime() + '\n')
    for adrr in ['E:', 'F:']:
        for file in os.listdir(adrr):
            dir = os.path.join(adrr, '\\' + file)
            # if os.path.isdir(dir) and 'music' in dir.lower(): # for music
            if os.path.isdir(dir) and 'DCIM' == file:
                searchFolder(dir, 1)
    # searchFolder(os.getcwd(), 2) # for music
    workbook.save(filename='D:\\PythonProject\\PythonSpider\\Calculate1.xls')
    print('\n<<< end ' + ctime())


from pymongo import MongoClient
from bson.json_util import dumps, loads
from dotenv import load_dotenv

import os

class Database(object):

    load_dotenv()
    url = os.getenv('MONGODB_CONNSTRING')
    database=None
    client=None

    @staticmethod
    def initialize():
        connection=MongoClient(Database.url)
        try:
            Database.client=connection
            Database.database = connection["textToSpeech"]
            print(' *', 'Connected to MongoDB!')
        except Exception as e:
            print(' *', "Failed to connect to MongoDB at")
            print('Database connection error:', e)

    @staticmethod
    def insert_one(collection, data):
        return Database.database[collection].insert_one(data)

    @staticmethod
    def close():
        Database.client.close()


import dash
from dash import dcc, html, Input, Output
import pandas as pd
import io
from dash.exceptions import PreventUpdate
import base64
import csv

external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']

new_df=Gold_coin_prediction.join([Universal_prediction, NewHope_prediction,CPI_prediction,Japfa_prediction,QLFeed_prediction,QLAgrofood_prediction,WidodoMakmur_prediction,Farmsco_prediction,SabasDian_prediction,KertamulyaSari_prediction,Agrico_prediction,Wonokoyo_prediction,CibadakIndah_prediction,Gorontalo_prediction,NTB_prediction])

app = dash.Dash(__name__,external_stylesheets=external_stylesheets)

app.layout = html.Div([
        html.H2('Feedmill Price Forecasts(daily)', style={'textAlign': 'center', 'font-family': 'Arial','fontWeight': 'bold'}),
        dcc.Graph(id="time-series-chart"),
        html.P("Select Feedmill:",style={'textAlign': 'center'}),
        dcc.Dropdown(
            id="ticker",
            options=["PT. Gold Coin","PT. Universal", "PT. NEW HOPE INDONESIA","CPI","Japfa","QLFeed","QLAgrofood","WidodoMakmur","Farmsco","SabasDian","KertamulyaSari","Agrico","Wonokoyo","CibadakIndah","Gorontalo","NTB"],
            value="PT. Gold Coin",
            clearable=False,
            style={'width': '50%', 'margin': 'auto'}
        ),
        html.Br(),
        html.Br(),
        html.Button('Download CSV', id='export-csv-button', n_clicks=0),
        dcc.Download(id="download-dataframe-csv"),
        html.Br(),
        html.Div(id='mae-div', style={'textAlign': 'center',
                                 'position': 'absolute',
                                  'bottom': '250px',
                                  'right': '70px'
                                     }),
        html.Br(),
        html.Div(id='mae-rolling-div', style={'textAlign': 'center',
                                 'position': 'absolute',
                                  'bottom': '220px',
                                  'right': '70px'
                                     })

    ])

@app.callback(
    Output("time-series-chart", "figure"),
    Input("ticker", "value"))
def display_time_series(ticker):
    filtered_df = new_df.loc[:, [ticker, f"yhat_{ticker}", f"y_lower_{ticker}", f"y_upper_{ticker}"]]
    filtered_df[f"yhat_{ticker}"] = filtered_df[f"yhat_{ticker}"].iloc[-7:]
    filtered_df[f"y_lower_{ticker}"] = filtered_df[f"y_lower_{ticker}"].iloc[-7:]
    filtered_df[f"y_upper_{ticker}"] = filtered_df[f"y_upper_{ticker}"].iloc[-7:]

    # Add rolling mean line
    rolling_mean = filtered_df[ticker].rolling(window=7, min_periods=1).mean().shift(7)
    fig = go.Figure()
    fig.add_trace(go.Scatter(x=filtered_df.index, y=filtered_df[ticker],
                mode='lines',
                name='Actual'))
    fig.add_trace(go.Scatter(x=filtered_df.index, y=rolling_mean,
                mode='lines',
                name='Rolling Mean'))
    fig.add_trace(go.Scatter(x=filtered_df.index, y=filtered_df[f"yhat_{ticker}"],
                mode='lines',
                name="Predicted"))
    fig.add_trace(go.Scatter(x=filtered_df.index, y=filtered_df[f"y_lower_{ticker}"],
                fill='tonexty',
                mode='lines',
                fillcolor='rgba(255, 0, 0, 0.2)',
                name="lower bound"))
    fig.add_trace(go.Scatter(x=filtered_df.index, y=filtered_df[f"y_upper_{ticker}"],
                fill='tonexty',
                mode='lines',
                fillcolor='rgba(0, 255, 0, 0.2)',
                name="upper bound"))
    fig.update_xaxes(title_text='Date',
    title_font=dict(
        family="Arial, sans-serif",
        size=18,
        color="black"))
    fig.update_yaxes(
    title_text='Feedmill Price',
    title_font=dict(
        family="Arial, sans-serif",
        size=18,
        color="black"))
    fig.update_layout(legend=dict(title="Legend", orientation="v", y=1.1, x=1.05, xanchor="center", bgcolor='rgba(0,0,0,0)'),
                     margin=dict(l=20, r=20, t=80, b=30)
                     )
    fig.update_traces(marker=dict(size=1.5))

    return fig

@app.callback(Output('download-dataframe-csv', 'data'),
              Input('export-csv-button', 'n_clicks'),
              Input('ticker', 'value'))
def download_csv(n_clicks, ticker):
    if n_clicks == 0:
        raise PreventUpdate
    filtered_df = new_df.loc[:, [ticker, f"yhat_{ticker}",f"y_lower_{ticker}",f"y_upper_{ticker}"]]
    filtered_df[f"{ticker}_rolling_mean"] = filtered_df[ticker].rolling(window=7, min_periods=1).mean().shift(7)
    filtered_df.reset_index(inplace=True)

    csv_string = io.StringIO()
    filtered_df.to_csv(csv_string, index=False, encoding='utf8')
    csv_string.seek(0)

    timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
    filename = f"{ticker}_daily_price_forecasts_{timestamp}.csv"

    return dict(filename=filename, content=csv_string.getvalue())

@app.callback(Output('mae-div', 'children'),
              Input('ticker', 'value'))
def display_mae(ticker):
    filtered_df = new_df.loc[:, [ticker, f"yhat_{ticker}"]]
    mae = np.mean(np.abs(filtered_df[ticker] - filtered_df[f"yhat_{ticker}"]))
    return html.Div([
        html.Div(f"Mean Absolute Error: {mae:.2f}"),
    ])

@app.callback(Output('mae-rolling-div', 'children'),
              Input('ticker', 'value'))
def display_rolling_mae(ticker):
    filtered_df = new_df.loc[:, [ticker, f"yhat_{ticker}"]]
    filtered_df[f"{ticker}_rolling_mean"] = filtered_df[ticker].rolling(window=7, min_periods=1).mean().shift(7) # change window size as per your requirement
    mae_rolling = np.mean(np.abs(filtered_df[f"{ticker}_rolling_mean"] - filtered_df[f"yhat_{ticker}"]))
    return html.Div([
        html.Div(f"Mean Absolute Error of Rolling Mean: {mae_rolling:.2f}")
    ])

if __name__ == '__main__':
    app.run_server(debug=True, use_reloader=False, port=8000, host='127.0.0.1')


from texttable import Texttable
import datetime
from _global import insert_message

class tabela_ambiente():
    def __init__(self, ambiente):
        self.ambiente = ambiente
        self.table = Texttable()

    def print_table(self, global_time):
        self.table.reset()
        self.table.set_deco(Texttable.HEADER)
        self.table.set_cols_dtype(['t',  # text
                              't',  # float
                            ])
        self.table.set_cols_align(["l", "c"])

        self.table.add_rows([["Informações do ambiente", ""],
                       ["Hora: ", str(datetime.timedelta(seconds=global_time))],
                       ["Temperatura: ", str(round(self.ambiente.temperatura, 2))],
                       ["Chuva: ", str(self.ambiente.chuva)],
                       ["Estado Atmosférico: ", str(self.ambiente.estado_atmosferico)],
                       ["Sujeira: ", str(self.ambiente.sujeira)],
                       [" ", " "],
                       ["Último movimento foi há: ", str(datetime.timedelta(seconds=self.ambiente.mov_count))],
                       ["Ar-condicionado: ", str(self.ambiente.ar_condicionado)],
                       ["Aquecedor: ", str(self.ambiente.aquecedor)],
                       ["Lâmpada: ", str(self.ambiente.lampada)],
                       ["Porta: ", str(self.ambiente.porta)],
                       ["Janela: ", str(self.ambiente.janela)],
                       ["Televisão: ", str(self.ambiente.televisão)],
                       ["Aspirador de pó: ", str(self.ambiente.aspirador)]])

        print(self.table.draw())

class tabela_tarefas():
    def __init__(self, lista_tarefas):
        self.lista = lista_tarefas
        self.table = Texttable()

    def print_table(self):
        self.table.reset()
        self.table.set_deco(Texttable.HEADER)
        self.table.set_cols_dtype(['t',  # text
                              't',  # text
                              't', #text
                              't',  #text
                              't' #text
                            ])
        self.table.set_cols_align(["l", "l", "c", "c", "c"])

        self.table.add_rows([["Fila de tarefas", "", "", "", ""],
                            ["ID: ", "Nome: ", "Deadline: ", "T. Execução: ", "T. Requerido"]]
                            )
        for tarefa in self.lista:
            self.table.add_row([str(tarefa.id), str(tarefa.nome), str(tarefa.deadline), str(tarefa.tempo_exec), str(tarefa.tempo_req)])

        print(self.table.draw())


import matplotlib.pyplot as plt
from time import sleep
import numpy as np
import pickle

fontsize = 11
fontname = "Arial"

plt.style.use("seaborn-whitegrid")
labels = [f"a{i}" for i in range(1,6)]
dist_1 = [0.05, 0.1, 0.025, 0.8, 0.025]
width = 0.35

x = np.arange(len(dist_1))

fig, ax = plt.subplots()
rects1 = ax.bar(x, dist_1, width, label='Team')

# Add some text for labels, title and custom x-axis tick labels, etc.
ax.set_ylabel('Probability', fontsize=30, fontname=fontname)
# ax.set_title('Accumulated Negative Rewards Misaligned Scenarios')
ax.set_xticks(x, labels, fontsize=30, fontname=fontname)
ax.set_ylim([0, 1])

# ax.bar_label(rects1, padding=3)
# ax.bar_label(rects2, padding=3)
# ax.bar_label(rects3, padding=3)

fig.tight_layout()
plt.savefig("plots/dist2.svg")
plt.show()


from pyquery import PyQuery
import requests
import datetime
from openpyxl import workbook
import re
# 进行数据的爬取
def sub_link():
    url = 'https://dt.8684.cn/bj'
    headers = {'User-Agent':
                    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36'
                }
    res = requests.get(url,headers = headers)
    #print('如果打印200则是请求成功:',res.status_code)
    start_time = datetime.datetime.now()
    print('开始爬取数据的时间为：',start_time)
    res.encoding = 'utf-8' # 对原网页的数据进行编码
    html_data = res.text
    subway_name = [] # 地铁
    subway_link = [] # 地铁的链接
    subway_name_list = [] # 最终的地铁路线名
    subway_link_list = []  #最终的地铁各个站台名
    doc =  PyQuery((''.join(html_data)))
    for item in doc.items('ul li a'):
        subway_name.append(item.text())
        subway_link.append(item.attr.href)
    # 对各个地铁进行处理
    subway_name = subway_name[:54]
    for i in range(1,len(subway_name)):
        if i % 2 != 0:
            subway_name_list.append(subway_name[i])
    # 对各个站台名进行处理
    subway_link = subway_link[:54]
    for i in range(1,len(subway_link)):
        if i % 2 != 0:
            subway_link_list.append(subway_link[i])
    # 发现地铁2号线和10号线与其他的结构是不一样的,所以把这两条线路单独放在一块,其他的线路放在一块
    subway_name_list_1 = [] # 除了2和10号线所有的地铁
    subway_link_1 = []
    subway_name_list_2_10 = []
    subway_link_2_10 = []
    for i in range(0, len(subway_link_list)):
        if i == 1 or i == 2 or i == 10 or i == 11:
            subway_name_list_2_10.append(subway_name_list[i])
            subway_link_2_10.append(subway_link_list[i])
        else:
            subway_name_list_1.append(subway_name_list[i])
            subway_link_1.append(subway_link_list[i])

    return subway_name_list_1,subway_link_1,subway_name_list_2_10,subway_link_2_10,start_time

def save(sub_name_1,sub_link_1,sub_name_2_10,subway_link_2_10):
    excel = workbook.Workbook() #创建一个excel文件
    sheet = excel.active
    # 先去求除2号线和10号线以后所有的地铁
    name_temp_1 = []
    name_1 = []
    station_name = [] # 总的站台

    star_time_1= [] # 首班车时间
    star_time_2 = []

    end_time_1 = [] # 尾班车时间
    end_time_2 = []

    station_name_2_10 = []

    station_temp_2_10 = [] # 2-10的站台名站台名
    str_time = []
    str_time_2_10 = [] # 2-10的时间

    final_station = []

    final_start_time_2_10 = [] # 最终的
    final_end_time_2_10 = []

    for k in range(0, len(sub_name_1)):  #len(sub_name_1)
        url = 'https://dt.8684.cn' + sub_link_1[k]
        station_1 = [] # 站台名
        time_total = []
        headers = {'User-Agent':
                    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36'
                }
        res = requests.get(url,headers = headers)
        #print('如果打印200则是请求成功:',res.status_code)
        res.encoding = 'utf-8' # 对原网页的数据进行编码
        html_data = res.text
        doc =  PyQuery((''.join(html_data)))

        for item in doc.items('div em'):
            name_temp_1.append(item.text())
        for item in doc.items('tr td a'):
            station_1.append(item.text())

        for item in doc.items('tr td'):
            time_total.append(item.text())
        str_time.append(time_total)
        station_name.append(station_1)
    for i in range(0, len(str_time)):
        star_temp_1= [] # 首班去的集合
        star_temp_2 = [] # 回来
        end_temp_1= []  # 尾班
        end_temp_2 = []
        for j in range(0, len(str_time[i])):
            if j % 5 == 1:
                star_temp_1.append(str_time[i][j])
            if j % 5 == 2:
                star_temp_2.append(str_time[i][j])
            if j % 5 == 3:
                end_temp_1.append(str_time[i][j])
            if j % 5 == 4:
                end_temp_2.append(str_time[i][j])
        star_time_1.append(star_temp_1) #  首班车去的时间
        star_time_2.append(star_temp_2)
        end_time_1.append(end_temp_1) # 尾班去的时间
        end_time_2.append(end_temp_2)

    for i in range(0,len(name_temp_1)):
        if i % 2 == 0:
            name_1.append(name_temp_1[i]) # name_1 地铁线路

    # 爬取剩余的2号线和10号线
    for r in range(0,len(sub_name_2_10)):
        url_1= 'https://dt.8684.cn' + subway_link_2_10[r]
        #print(url_1)

        time_total_2_10 = []
        headers = {'User-Agent':
                    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36'
                }
        res_1= requests.get(url_1,headers = headers)
        #print('如果打印200则是请求成功:',res.status_code)
        res_1.encoding = 'utf-8' # 对原网页的数据进行编码
        html_data_1 = res_1.text
        doc_1=  PyQuery((''.join(html_data_1)))
        for item in doc_1.items('div em'): # 站台名
            station_temp_2_10.append(item.text())
        for item in doc_1.items('tr td'):
            time_total_2_10.append(item.text())
        str_time_2_10.append(time_total_2_10)
    #print(str_time_2_10[0])
    #print(str_time_2_10[2])
    for i in range(0, len(str_time_2_10)): # 4
        station_2_10 = [] # 站台
        final_start_time_2 = [] # 开始时间
        final_end_time_2 = [] # 结束时间
        for j in range(0, len(str_time_2_10[i])):
            if j % 3 == 0:
                station_2_10.append(str_time_2_10[i][j])
            if j % 3 == 1:
                final_start_time_2.append(str_time_2_10[i][j])
            if j % 3 == 2:
                final_end_time_2.append(str_time_2_10[i][j])
        final_station.append(station_2_10) # 2,10号线的站名
        final_start_time_2_10.append(final_start_time_2) # 2,10 开始时间
        final_end_time_2_10.append(final_end_time_2) # 2,10 结束时间
    for i in range(1,len(station_temp_2_10)):
        if i % 3== 0:
            station_name_2_10.append(station_temp_2_10[i])

    for i in range(0,len(name_1)):
        sheet.cell(1, 5*(i+1)-4).value = name_1[i]
        sheet.cell(1, 5*(i+1)-3).value = '首班车'
        sheet.cell(1, 5*(i+1)-2).value = '首班车'
        sheet.cell(1, 5*(i+1)-1).value = '尾班车'
        sheet.cell(1, 5*(i+1)).value = '尾班车'
    #print(sheet.max_column)
    for i in range(0,len(station_name)): # len(station_name) = 23
        for j in range(0,len(station_name[i])):
            sheet.cell(j + 2,5*(i+1)-4).value = station_name[i][j] # 站台名

    for i in range(0,len(star_time_1)):
        for j in range(0,len(star_time_1[i])):
            sheet.cell(j+2,5*(i+1)-3).value = star_time_1[i][j]
            sheet.cell(j+2,5*(i+1)-2).value = star_time_2[i][j]
            sheet.cell(j+2,5*(i+1)-1).value = end_time_1[i][j]
            sheet.cell(j+2,5*(i+1)).value = end_time_2[i][j]

    '''对2号线和10号线的写入以及爬取可以进一步的优化'''

    # 前面一共115列
    # 将2号线和10号线写入文件中
    sheet.cell(1,116).value = sub_name_2_10[0]
    sheet.cell(1,117).value = '首班车'
    sheet.cell(1,118).value = '尾班车'
    sheet.cell(1,119).value = sub_name_2_10[1]
    sheet.cell(1,120).value = '首班车'
    sheet.cell(1,121).value = '尾班车'
    sheet.cell(1,122).value = sub_name_2_10[2]
    sheet.cell(1,123).value = '首班车'
    sheet.cell(1,124).value = '尾班车'
    sheet.cell(1,125).value = sub_name_2_10[3]
    sheet.cell(1,126).value = '首班车'
    sheet.cell(1,127).value = '尾班车'
    # final_station:2,10号线的站名
    # final_start_time_2_10:2,10 开始时间
    # final_end_time_2_10:2,10 结束时间
    for i in range(0,len(final_station)):
        for j in range(0,len(final_station[i])):
            sheet.cell(j+2,3*(i+1)+113).value = final_station[i][j]
    for i in range(0,len(final_start_time_2_10)):
        for j in range(0,len(final_start_time_2_10[i])):
            sheet.cell(j+2,3*(i+1)+114).value = final_start_time_2_10[i][j]
    for i in range(0,len(final_end_time_2_10)):
        for j in range(0,len(final_end_time_2_10[i])):
            sheet.cell(j+2,3*(i+1)+115).value = final_end_time_2_10[i][j]

    excel.save(r'D:/bj_subway.xlsx')
if __name__ == "__main__":
    print('开始爬取,################保持微笑,不要出错#############')
    sub_name_1,sub_link_1,sub_name_2_10,subway_link_2_10,start_time= sub_link()
    save(sub_name_1,sub_link_1,sub_name_2_10,subway_link_2_10)
    cur_time = datetime.datetime.now()
    print('爬取结束的时间为:',cur_time)
    cost_time = cur_time - start_time
    print('爬取共用的时间为:',cost_time)
    print('终于结束,可以大笑!！！！！！！！！！！！！！！！！！！')


# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import unittest

import yaml

from fbcli import editor

class TestText(unittest.TestCase):

    def test_meta_and_body(self):
        txt = '''Title: Valid title
Project: Proj
Area: misc
Assign to: me
Priority: high
---
This is the body'''
        t = editor.Text(txt)
        self.assertEqual(t.meta['Title'], 'Valid title')
        self.assertEqual(t.meta['Project'], 'Proj')
        self.assertEqual(t.meta['Area'], 'misc')
        self.assertEqual(t.meta['Assign to'], 'me')
        self.assertEqual(t.meta['Priority'], 'high')

    def test_invalid_yaml(self):
        txt = '''Title:Invalid because no space after column
Project:Proj
Area:misc
Assign to: me
Priority: for consideration
---
This is the body'''
        t = editor.Text(txt)
        with self.assertRaises(yaml.error.YAMLError):
            t.meta  # pylint: disable=pointless-statement

    def test_utf8_body(self):
        txt = '''Title: Some ¢hars
Project: Proj
Area: misc
Assign to: me
Priority: high
---
This is the body with mo®e utf8.
'''
        t = editor.Text(txt)
        self.assertEqual(t.meta['Title'], 'Some ¢hars')
        self.assertEqual(t.body, 'This is the body with mo®e utf8.')

    def test_body_with_comments(self):
        txt = '''Title: title title
Project: Proj
Area: misc
Assign to: me
Priority: high
---
# This comment should stay
This is the body.
# This comment should stay, too

# This comment won't be included because there are no blank lines
# Lines starting wth "#" will be ignored.
# Leave this file empty to abort action.
# It's possible to add metadata in the format of a header.
# Use "---" as separator between the header and the body.
# E.g. To upload files use:
#    Files:
#      - path_to_file_1
#      - path_to_file_2
'''
        expected = '''# This comment should stay
This is the body.
# This comment should stay, too'''
        t = editor.Text(txt)
        self.assertEqual(t.body, expected)

    def test_new_invalid_title(self):
        txt = '''Title: <title>
Project: Proj
Area: misc
Assign to: me
Priority: for consideration
---
This is the body'''
        t = editor.Text(txt)
        with self.assertRaises(AssertionError):
            t.get_params_for_new()

    def test_new_empty_title(self):
        txt = '''Title:
Project: Proj
Area: misc
Assign to: me
Priority: for consideration
---
This is the body'''
        t = editor.Text(txt)
        with self.assertRaises(AssertionError):
            t.get_params_for_new()

class TestEditor(unittest.TestCase):

    def test_write_new(self):
        editor.clear()
        expected = '''
# Lines starting wth "#" will be ignored.
# Leave this file empty to abort action.
# It's possible to add metadata in the format of a header.
# Use "---" as separator between the header and the body.
# E.g. To upload files use:
#    Files:
#      - path_to_file_1
#      - path_to_file_2
'''

        with editor.writing():
            with open(editor.FNAME, 'r') as fid:
                body = fid.read()
            self.assertEqual(body, expected)


import urllib.request
import ssl

ssl._create_default_https_context = ssl._create_unverified_context

import urllib, http.cookiejar

opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar()))
# then for all requests
postData = ""
if postData:
    pData =  urllib.parse.urlencode(postData)
else:
    pData = None

httpReq = urllib.request.Request("https://userscloud.com/2rdteo81ocgs", pData)
page =  opener.open(httpReq)
strPage = page.read()
strPage = strPage.decode()
print(strPage)
with open("out.html", 'a') as out:
    out.write(strPage + '\n')

posRan = strPage.find("\"rand\"")
print(posRan)

#import requests
#url_0 = "http://dailyuploads.net/030p4rn9ll6a"
#url = "https://webapp.pucrs.br/consulta/servlet/consulta.aluno.ValidaAluno"
#data = {"pr1": "123456789", "pr2": "1234"}

#s = requests.session()
#r = s.get(url_0)

#print(r)


from os import environ
from dotenv import load_dotenv

load_dotenv()

# Django Config
SECRET_KEY = environ.get("SECRET_KEY", 'secret-key')

# News API https://newsapi.org/
NEWS_API_KEY = environ.get("NEWS_API_KEY")
COUNTRIES = ["us", "in", "gb"]
BASE_NEWS_URL = f"https://newsapi.org/v2/top-headlines?apiKey={NEWS_API_KEY}"
CATEGORIES = ["entertainment", "science", "technology"]

# Database
# DATABASE_URL = environ.get("DATABASE_URL")

# Sendgrid E-mail
SENDGRID_API = environ.get("SENDGRID_API")
SENDGRID_FROM_MAIL = environ.get("SENDGRID_FROM_MAIL")

# Redis Server
REDIS_URL = environ.get("REDIS_URL")


import re
import json
import argparse
import numpy as np
import pandas as pd
from pathlib import Path

def build_label_dict(questions_path, category = None):
    questions = pd.read_csv(open(questions_path, 'r'), quotechar='"')
    if category is not None:
        questions = questions.loc[questions['Category'] == category]
    questions = set(questions['Answer'].apply(lambda x: np.array(re.sub(r'\s+', '_', x))))
    return dict(zip(questions, list(range(len(questions)))))

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="""
        A simple program to create label dictionary used for converting label to ID and vice versa.
    """)
    parser.add_argument('--questions-path', help='Path to question dataset.', default='.\\data\\questions.csv')
    parser.add_argument('--category', help='Category of questions. If not provided all categories will be considered.')
    parser.add_argument('--output-path', help='Path to save labels dictionary to', default='.\\data\\answers.json')
    args = parser.parse_args()
    question_to_id = build_label_dict(args.questions_path, category=args.category)
    json.dump(question_to_id, open(args.output_path, 'w', encoding='utf8'))


from bokeh.plotting import figure, output_file, show
from bokeh.io import output_notebook, show
from bokeh.models.tickers import FixedTicker
from bokeh.models import NumeralTickFormatter, Legend
import pandas as pd
import numpy as np
import bokeh.palettes

def extract_dfs(df):
	"""
	df: A pandas dataframe where:
		- The index (rows) contains years or other temporal period
		- The columns represent categories, countries etc
		- If the rows are years and the columns are countries (for example)
			then an entry in the dataframe represents whichever quantity is
			being measured (e.g. GDP) for that country and year.
		(For more details view the example.ipynb)
	returns:
		- Dataframe with total interannual percentage growth
		- Datframe with contributions to growth of each original column
	"""
	dff = df.copy()
	categories = list(dff.columns)
	dff['total'] = dff.sum(axis=1)
	dff['pctg_growth_total'] = (dff['total'] - dff['total'].shift(1)) / dff['total'].shift(1)
	for cat in categories:
	    dff[f'pctg_growth_{cat}'] = (dff[cat] - df[cat].shift(1)) / dff[cat].shift(1)
	for cat in categories:
	    dff[f'contrib_{cat}'] = dff[f'pctg_growth_{cat}']*(dff[cat].shift(1)) / dff['total'].shift(1)
	growth_df = dff[['pctg_growth_total']]
	growth_df.columns = ['total']

	contrib_df = dff[[f'contrib_{cat}' for cat in categories]]
	contrib_df.columns = categories
	return growth_df.iloc[1:], contrib_df.iloc[1:]

def plot_growth(df, period='year', bar_width=0.5):
	"""
	df: A pandas dataframe (typically the first output of extract_dfs)
	period: string, the periodicity of the data
	bar_width: float, width of the bars in the plot

	The function plots the overall percentage interannual growth
	"""

	# output to static HTML file
	output_file(f'{period}_growth.html')

	# create a new plot
	p = figure(tools='pan,box_zoom,reset,save',
	           title=f'{period} to {period} growth (total)'.capitalize(),
	           x_axis_label=f'{period}'.capitalize())

	p.vbar(df.index, top=df['total'], width=bar_width)
	p.yaxis[0].formatter = NumeralTickFormatter(format="0%")
	p.xaxis.ticker = FixedTicker(ticks=list(df.index))
	show(p)

def tops_bottoms(df):
	"""
	df: A pandas dataframe (typically the second output of extract_dfs)

	Using the input dataframe, the function returns two dataframes that
	contain the bottom and top limits of bar segments in a stacked bar chart
	"""
	dfa = np.array(df)
	tops = np.zeros(dfa.shape)
	bots = np.zeros(dfa.shape)

	for i in range(tops.shape[0]):
	    neg = 0
	    pos = 0
	    for j in range(tops.shape[1]):
	        if dfa[i, j] >= 0:
	            bots[i, j] = pos
	            pos += dfa[i, j]
	            tops[i, j] = pos
	        else:
	            tops[i, j] = neg
	            neg += dfa[i, j]
	            bots[i, j] = neg
	tops_df = pd.DataFrame(tops)
	tops_df.columns = df.columns
	tops_df.index = df.index
	bots_df = pd.DataFrame(bots)
	bots_df.columns = df.columns
	bots_df.index = df.index
	return tops_df, bots_df

def plot_contrib(df, palette='Colorblind', period='year', bar_width=0.5, legend_loc='top_right'):
	"""
	df: A pandas dataframe (typically the second output of extract_dfs)
	palette: string, the name of a bokeh color palette to be applied to the plot
	period: string, the periodicity of the data
	bar_width: float, width of the bars in the plot
	legend_loc: string, location of the legend in the plot e.g: 'bottom_left'

	The function plots the contributions of each column to the overall
	interannual growth
	"""
	categories = list(df.columns)
	colors = eval(f'bokeh.palettes.{palette}')[len(categories)]
	tops, bottoms = tops_bottoms(df)

	# output to static HTML file
	output_file(f'Growth_contribs.html')

	# create a new plot
	p = figure(tools='pan,box_zoom,reset,save',
	           title='Contributions to total growth',
	           x_axis_label=f'{period}'.capitalize())

	for i in range(len(categories)):
	    p.vbar(df.index, bar_width, bottom=bottoms[categories[i]], top=tops[categories[i]],
	          color=colors[i], legend=categories[i])
	p.yaxis[0].formatter = NumeralTickFormatter(format="0%")
	p.xaxis.ticker = FixedTicker(ticks=list(df.index))
	p.legend.location = legend_loc
	show(p)

def plot_contributions(df, palette='Colorblind', period='year', bar_width=0.5,
	legend_loc='top_right', total_growth=True):
	"""
	df: A pandas dataframe with the appropriate format (see example.ipynb)
	palette: string, the name of a bokeh color palette to be applied to the plot
	period: string, the periodicity of the data
	bar_width: float, width of the bars in the plot
	legend_loc: string, location of the legend in the plot e.g: 'bottom_left'
	total_growth: bool, whether to plot total period to period growth

	This function plots:
		- Percentage total period to period growth (optional)
		- Contributions to growth of each column in the dataframe
	"""
	growth, contribs = extract_dfs(df)
	if total_growth:
		plot_growth(growth, period=period, bar_width=bar_width)
	plot_contrib(contribs, palette=palette, period=period, bar_width=bar_width, legend_loc=legend_loc)


from sqlalchemy import create_engine, text
import pandas as pd

SQLALCHEMY_DATABASE_URL = "mysql://root:    @localhost:3306/ecommerce_db"

engine = create_engine(SQLALCHEMY_DATABASE_URL)
con = engine.connect()

category_insert_statement = text(
    """
INSERT INTO category (id, name, `desc`)
VALUES
  (1, 'cat1', 'test desc'),
  (2, 'cat2', 'test desc'),
  (3, 'cat3', 'test desc'),
  (4, 'cat4', 'test desc');
"""
)

con.execute(category_insert_statement)
con.commit()

product_insert_statement = text(
    """
INSERT INTO product (id, name, `desc`, category_id, price)
VALUES
  (1, 'prod1', 'test desc', 1, 10.0),
  (2, 'prod2', 'test desc', 2, 20.0),
  (3, 'prod3', 'test desc', 3, 30.0),
  (4, 'prod4', 'test desc', 4, 40.0);
"""
)
con.execute(product_insert_statement)
con.commit()

inventory_insert_statement = text(
    """
INSERT INTO inventory (id, product_id, current_stock, low_stock_alert_threshold)
VALUES
    (1, 1, 10, 1),
    (2, 2, 20, 2),
    (3, 3, 30, 3),
    (4, 4, 40, 4);
"""
)
con.execute(inventory_insert_statement)
con.commit()

sales_insert_statement = text(
    """
INSERT INTO sales (id)
VALUES
    (1),
    (2),
    (3),
    (4);
"""
)
con.execute(sales_insert_statement)
con.commit()

sale_items_insert_statement = text(
    """
INSERT INTO sale_items (id, sales_id, product_id, quantity)
VALUES
    (1, 1, 1, 3),
    (2, 1, 4, 1),
    (3, 2, 1, 3),
    (4, 2, 3, 6),
    (5, 2, 2, 5),
    (6, 2, 4, 2),
    (7, 3, 4, 6),
    (8, 3, 1, 2),
    (9, 3, 3, 5),
    (10, 4, 2, 4),
    (11, 4, 1, 3);
"""
)
con.execute(sale_items_insert_statement)
con.commit()

con.close()


from rest_framework import serializers

from core.models import Disease, DiseaseInfectionStatus

class DiseaseInfectionStatusListSerializer(serializers.ModelSerializer):
    """
    DiseaseInfectionStatusListSerializer

    Serializes and outputs infection status
    """
    id = serializers.CharField(read_only=True)
    desc = serializers.CharField(source='infection_status')

    class Meta:
        model = DiseaseInfectionStatus
        fields = ('id', 'desc')

class DiseaseSerializer(serializers.ModelSerializer):
    """
    Serializes disease information including all the infection status associated with it.
    """
    id = serializers.CharField(read_only=True)
    infection_status = DiseaseInfectionStatusListSerializer(many=True, source='diseaseinfectionstatus_set',
                                                            read_only=True)

    class Meta:
        model = Disease
        fields = ('id', 'name', 'infection_status')

class DiseaseInfectionStatusDetailSerializer(serializers.Serializer):
    """
    DiseaseInfectionStatusDetailSerializer

    Serializes the disease infection status create and update data
    """
    id = serializers.CharField(read_only=True)
    infection_status = serializers.CharField(max_length=255)

    def validate_disease_id(self, disease_id):
        """
        Validates the given disease ID, checks if an associated disease exists !

        :param disease_id:
        :return:
        """
        try:
            Disease.objects.get(id=disease_id)
        except Disease.DoesNotExist:
            raise serializers.ValidationError('Please provide a valid disease ID !')
        return disease_id

    def create(self, validated_data):
        """
        Creates a disease record

        :param validated_data:
        :return:
        """
        disease = validated_data.pop("disease")
        return DiseaseInfectionStatus.objects.create(infection_status=validated_data.get('infection_status'),
                                                     disease=disease)

    def update(self, instance, validated_data):
        """
        Updates a disease record

        :param instance:
        :param validated_data:
        :return:
        """
        instance.infection_status = validated_data.get('infection_status', instance.infection_status)
        instance.save()

        return instance


import os
import cv2
import argparse
import numpy as np
from glob import glob
from tqdm import tqdm
from PIL import Image
from pathlib import Path
from xml.dom import minidom
from natsort import natsorted
from dataclasses import dataclass
from Utils import rotate_image, create_dir, group_lines, preprocess_img, parse_labels
from IPython.display import Image as ShowImage

@dataclass
class LineSample:
    """Class for keeping track of an item in inventory."""
    image: np.array
    label: str
    x: float
    y: float
    width: float
    height: float

def generate_line_image_v1(image, contour, angle: float, kernel: tuple = (10, 16), iterations: int = 6):
    image_mask = np.zeros((image.shape[0], image.shape[1]), dtype=np.uint8)
    cv2.drawContours(image_mask, [contour], contourIdx=-1, color=(255, 255, 255), thickness=-1)

    dilate_k = np.ones(kernel, dtype=np.uint8)
    kernel_iterations = iterations

    image_mask = cv2.dilate(image_mask, dilate_k, iterations=kernel_iterations)
    image_masked = cv2.bitwise_and(image, image, mask=image_mask)

    if angle > 85.0 and angle != 90.0:
        angle = -(90 - angle)

    if angle == 90:
        angle = 0

    rotated_img = rotate_image(image_masked, angle=angle)

    cropped_img = np.delete(rotated_img, np.where(~rotated_img.any(axis=1))[0], axis=0)
    cropped_img = np.delete(cropped_img,np.where(~cropped_img.any(axis=0))[0], axis=1)

    return cropped_img

def blurr(image: np.array, blur_intensity: int = 4):
    bw = cv2.GaussianBlur(image, (blur_intensity, blur_intensity), 0)
    return bw

def adaptive_binarize(image: np.array, block_size: int = 13, c: float = 11, invert: bool = False):
    c = round(c, 2)

    bw = cv2.adaptiveThreshold(image, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, block_size, c)

    if invert:
        bw = cv2.bitwise_not(bw)

    return bw

def get_components(image: np.array):
    connectivity = 4 # or 8, check here: https://stackoverflow.com/questions/7088678/4-connected-vs-8-connected-in-connected-component-labeling-what-is-are-the-meri
    image = cv2.bitwise_not(image)
    output = cv2.connectedComponentsWithStats(image, 4, cv2.CV_32S)
    numLabels, labels, stats, centroids = output

    return numLabels, labels, stats, centroids

def get_component_info(index: int, stats, centroids):

    x = stats[index, cv2.CC_STAT_LEFT]
    y = stats[index, cv2.CC_STAT_TOP]
    w = stats[index, cv2.CC_STAT_WIDTH]
    h = stats[index, cv2.CC_STAT_HEIGHT]
    area = stats[index, cv2.CC_STAT_AREA]
    (cX, cY) = centroids[index]
    return x, y, w, h, cX, cY, area

def filter_components(image: np.array, y_offset: int = 3, area_threshold: int = 6, y_border: bool = True, x_border: bool = False, filter_area: bool = True):

    mask = np.zeros(image.shape, dtype="uint8")
    numLabels, labels, stats, centroids = get_components(image)

    for compnt_idx in range(0, numLabels):
        x, y, w, h, cX, cY, area = get_component_info(compnt_idx, stats, centroids)

        # component touches the border
        x_pos = x > 0 and x+w < image.shape[1] - 1
        y_pos = y > 0 and y+h < image.shape[0] - 1
        #y_pos = y > 0

        cy_filter = cY > y_offset and cY < image.shape[0] - y_offset
        area_size = area > area_threshold

        filters = []
        filters.append(cy_filter)

        if x_border:
            filters.append(x_pos)

        if y_border:
            filters.append(y_pos)

        if filter_area:
            filters.append(area_size)

        if all(filters):
            componentMask = (labels == compnt_idx).astype("uint8") * 255
            mask = cv2.bitwise_or(mask, componentMask)

    mask = cv2.bitwise_not(mask)
    return mask

def get_page_data(image: str, annotation: str) -> tuple[str, list[LineSample]]:

    image = cv2.imread(image)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    image = blurr(image, blur_intensity=3)
    image = cv2.dilate(image, (3,3), 3)
    image = adaptive_binarize(image, invert=True, block_size=11, c=11)

    annotation_tree = minidom.parse(annotation)
    textlines = annotation_tree.getElementsByTagName('TextLine')
    centers, contour_dict = parse_labels(textlines, y_offset=0)

    page_data: list[LineSample] = []

    for _, (_, (k, v)) in enumerate(zip(centers, contour_dict.items())):
        points, label, angle = v

        if len(label) > 30:
            (x, y), (width, height), angle = cv2.minAreaRect(points)
            line_image = generate_line_image_v1(image, points, angle, kernel=(6, 4), iterations=14)
            line_image = cv2.bitwise_not(line_image)
            y_off = int(line_image.shape[0] * 0.2)
            line_image = filter_components(line_image, y_offset=y_off, area_threshold=10, x_border=False)

            line_sample = LineSample(line_image, label, x, y, width, height)
            page_data.append(line_sample)

    return page_data

def save_line_transcription(file_name, index, image, label, images_out, labels_out):
    line_img_path = os.path.join(images_out, f"{file_name}_{index}.jpg")
    cv2.imwrite(line_img_path, image)

    labe_file_path = os.path.join(labels_out, f"{file_name}_{index}.txt")

    with open(labe_file_path, "w", encoding="utf-8") as f:
        f.write(label)

if __name__ == "__main__":
    volumes = ["W2KG229028-v1", "W2KG229028-v2", "W2KG229028-v3", "W2KG229028-v4", "W2KG229028-v5", "W2KG229028-v6", "W2KG229028-v7", "W2KG229028-v8", "W2KG229028-v9", "W2KG229028-v10", "W2KG229028-v14", "W2KG229028-v15","W2KG229028-v17", "W2KG229028-v20", "W2KG229028-v21", "W2KG229028-v26", "W2KG229028-v28", "W2KG229028-v30"]

    input_dir = "D:/Datasets/Tibetan/Glomanthang/Annotations_v2/Glomanthang-Annotations"
    dataset_out = os.path.join(input_dir, "Dataset_clean")
    dataset_img_out = os.path.join(dataset_out, "lines")
    dataset_transcriptions_out = os.path.join(dataset_out, "transcriptions")

    create_dir(dataset_out)
    create_dir(dataset_img_out)
    create_dir(dataset_transcriptions_out)

    for volume_dir in volumes:

        dataset_images = natsorted(glob(f"{input_dir}/{volume_dir}/*.jpg"))
        dataset_labels = natsorted(glob(f"{input_dir}/{volume_dir}/page/*.xml"))

        print(f"Volume: {volume_dir} => Images: {len(dataset_images)} , Labels: {len(dataset_labels)}")

        for image, annotation in tqdm(zip(dataset_images, dataset_labels), total=len(dataset_images)):
            file_name = os.path.basename(image).split(".")[0]
            page_data = get_page_data(image, annotation)

            for idx, line in enumerate(page_data):
                #print(line.label)
                save_line_transcription(file_name, idx, line.image, line.label, dataset_img_out, dataset_transcriptions_out)


from icalendar import Calendar, Event
from datetime import date, datetime, time, timedelta
import dateutil.parser
from dateutil import relativedelta
import pytz
import json
import pprint
import csv

class Parser:
    def __init__(self, file):
        self.data = self.parse_ical(file)

    def parse_ical(self, flocation):
        cet = pytz.country_timezones('ch')

        g = open(flocation, 'rb')

        arr = []

        gcal = Calendar.from_ical(g.read())
        for component in gcal.walk():
            obj = {}
            keys = ['DTEND', 'DTSTART',
                    'ATTENDEE', 'LOCATION', "MY_PARTSTAT", "SUMMARY", "STATUS"]
            # Get Vevent as this holds all of the individual calender items
            if component.name == "VEVENT":
                # iterate over all of the keys - this is important as not every item has the same keys
                for k in component.keys():
                    if k in keys:
                        # get all the dates and convert them to strings
                        if k == 'DTEND' or k == 'DTSTAMP' or k == 'DTSTART' or k == 'LAST-MODIFIED' or k == 'CREATED':
                            new_dt = str(component.get(k).dt)[:19]
                            obj[k] = dateutil.parser.isoparse(new_dt)
                        # Locations is used by zoom to store a zoom link
                        # If there is a zoom link just append zoom otherwise add location to array
                        elif k == 'LOCATION':
                            obj[k] = []
                            for l in component.get(k).split(", "):
                                if "zoom.us" in str(l) and len(component.get(k).split(", ")) == 1:
                                    obj[k].append("zoom")
                        # Clean attendee list by only storing email address
                        elif k == 'ATTENDEE':
                            obj[k] = []
                            for e in component.get(k):
                                if e.replace("mailto:", "") == str(gcal["X-WR-CALNAME"]) and hasattr(e, 'params'):
                                    # print("-", e.replace("mailto:", ""))
                                    obj["MY_PARTSTAT"] = str(
                                        e.params["PARTSTAT"])
                                    obj[k].append(e.replace("mailto:", ""))
                                elif len(e) > 1:
                                    # print("-", e.replace("mailto:", ""))
                                    obj[k].append(e.replace("mailto:", ""))

                        else:
                            obj[k] = str(component.get(k))

            arr.append(obj)

        g.close()
        return arr

    def get_data(self):
        return self.data

    def get_timedelta(self, dtstart, dtend):
        timedelta = []
        # convert date string to date object
        sdate = datetime.strptime(dtstart, '%Y-%m-%d')
        edate = datetime.strptime(dtend, '%Y-%m-%d')
        # iterate over parsed calendar events and retrun if they are between two dates
        for e in self.data:
            if "DTSTART" in e:
                 # e["DTSTART"][:10] is shorting date string such as '2020-10-13 12:45:00+00:00' to '2020-10-13'
                timestr = datetime.strptime(
                    str(e["DTSTART"])[: 10], '%Y-%m-%d')
                if "DTSTART" in e and timestr.date() >= sdate.date() and timestr.date() <= edate.date():
                    timedelta.append(e)
        return timedelta

    def save_csv(self, arr):
        csv_file = "cal_data.csv"
        csv_columns = ['DTEND', 'DTSTART',
                       'ATTENDEE', 'LOCATION', "MY_PARTSTAT", "SUMMARY", "STATUS"]
        try:
            with open(csv_file, 'w') as csvfile:
                writer = csv.DictWriter(csvfile, fieldnames=csv_columns)
                writer.writeheader()
                for data in arr:
                    writer.writerow(data)

        except IOError:
            print("I/O error")

    # timeSum = timedelta(0)

test_cal = Parser('/Users/Simon/Downloads/test/test2.ics')

# for i in arr:
#     if "duration" in i:
#         timeSum = timeSum + i["duration"]

# def counts(my_list):
#     meeting_totals = {}
#     for meeting in my_list:
#         if "duration" in meeting and "ACCEPTED" in meeting.values():
#             meeting_totals[meeting["summary"]] = meeting_totals.get(
#                 meeting["summary"], 0) + 1
#     return meeting_totals

# def sum_time(my_list):
#     meeting_totals = {}
#     for meeting in my_list:
#         if "duration" in meeting and "ACCEPTED" in meeting.values():
#             meeting_totals[meeting["summary"]] = meeting_totals.get(
#                 meeting["summary"], timedelta(0)) + meeting["duration"]
#     return meeting_totals

# def sum_attendees(my_list, atCount=None):
#     meeting_totals = {}
#     for meeting in my_list:
#         if "duration" in meeting and "ACCEPTED" in meeting.values():
#             if atCount is None:
#                 for at in meeting["attendees"]:
#                     meeting_totals[at] = meeting_totals.get(
#                         at, 0) + 1
#                     print(at)
#             else:
#                 if len(meeting["attendees"]) <= atCount:
#                     print(meeting["attendees"])
#                     for at in meeting["attendees"]:
#                         meeting_totals[at] = meeting_totals.get(
#                             at, 0) + 1
#     return meeting_totals

# def count_by_attendees(my_list, atCount=None):
#     meeting_totals = {}
#     for meeting in my_list:
#         if "duration" in meeting and "ACCEPTED" in meeting.values():
#             if atCount is None:
#                 meeting_totals[meeting["summary"]] = meeting_totals.get(
#                     meeting["summary"], 0) + 1
#             else:
#                 if len(meeting["attendees"]) <= atCount:
#                     meeting_totals[meeting["summary"]] = meeting_totals.get(
#                         meeting["summary"], 0) + 1

#     return meeting_totals

# arr1 = counts(arr)
# arr2 = sum_time(arr)
# arr3 = sum_attendees(arr, 1)
# arr4 = count_by_attendees(arr, 5)
# print(arr3)

# today = datetime.utcnow()

# def return_before_after(my_list, dtstart, dtend):
#     newArr = []
#     sdate = datetime.strptime(dtstart, '%Y-%m-%d')
#     edate = datetime.strptime(dtend, '%Y-%m-%d')
#     for e in my_list:
#         if "DTSTART" in e and hasattr(e["DTSTART"], "date") and e["DTSTART"].date() >= sdate.date() and e["DTSTART"].date() <= edate.date():
#             print(e["summary"], e["DTSTART"].date(),
#                   e["DTSTART"].date() > sdate.date())
#             newArr.append(e)
#     return newArr

# largeMeetings = count_by_attendees(arr, 3)

# arrBA = return_before_after(largeMeetings, "2020-01-01", "2020-09-25")

# print(arrBA)
# print(sum_time(arrBA))

test_cal.save_csv(test_cal.get_timedelta("2020-01-01", "2020-09-25"))
# pprint.pprint(test_cal.data)

print(len(test_cal.get_timedelta("2020-01-01", "2020-09-25")))

# --------------------------
# with open('cal.json', 'w') as outfile:
#     json.dump(arr, outfile)

# ------------------------------
# for key in arr1:
#     print(key, "|", arr1[key], "|", arr2[key])

# if "duration" in meeting and "ACCEPTED" in meeting.values():
#             if meeting_totals[meeting["summary"]] is None:
#                 meeting_totals[meeting["summary"]] = [1, meeting["duration"]]
#             else:
#                 meeting_totals[meeting["summary"]] = [meeting_totals.get(
#                     meeting["summary"][0]) + 1, meeting_totals.get(
#                     meeting["summary"][1]) + meeting["duration"]]