from openalpr import Alpr
import cv2
import sys
import os
import time
import csv
from datetime import datetime
import random
#from filter
face_cascade = cv2.CascadeClassifier('data/haarcascades/haarcascade_frontalface_default.xml')

name=['car.mp4','car1.mp4']
window=['cam1','cam2']
cap = [cv2.VideoCapture(i) for i in name]
time.sleep(1)

time.sleep(1)
alpr = Alpr("in", "openalpr.conf", "runtime_data")
if not alpr.is_loaded():
    print("Error loading OpenALPR")
    sys.exit(1)
alpr.set_top_n(20)
dst=os.getcwd()+'\cropedImages'         # destination to save the images
if not os.path.exists(dst):
    os.makedirs(dst)

destination=os.getcwd()+'\Images'         # destination to save the images
if not os.path.exists(destination):
    os.makedirs(destination)

fieldnames = [ 'Plate number','ID', 'Image Path','Date and time ' ,'Confidance','window']
fileID=open('file.csv','w')       # file to save the data
newFileWriter = csv.writer(fileID)
newFileWriter.writerow(fieldnames)
print("Using OpenALPR " + alpr.get_version())

bg_subtractor = cv2.createBackgroundSubtractorMOG2(
    history=500, detectShadows=False)

print ('Training BG Subtractor-')

frames = [None] * len(name)
faces = [None] * len(name)
gray = [None] * len(name)
ret = [None] * len(name)
Rfilter= [None] * len(name)
filtered=[None] * len(name)
edged=[None] * len(name)
fg_mask=[None] * len(name)
enc=[None] * len(name)
cropedframe=[None] * len(name)
roi_color=[None] * len(name)
x=[None] * len(name)
y=[None] * len(name)
h=[None] * len(name)
w=[None] * len(name)
cnt=0
count=0
while True:

    for i,c in enumerate(cap):
        if c is not None:
            ret[i], frames[i] = c.read()
    for i,f in enumerate(frames):
        #s=True
        if ret[i] is True :
            #print(type(ret[i]))
            gray[i] = cv2.cvtColor(f, cv2.COLOR_BGR2GRAY)
            Rfilter[i] = cv2.bilateralFilter(gray[i], 5, 70, 70)
            faces[i]=face_cascade.detectMultiScale(gray[i], 1.3, 5)
##
##            for (x,y,w,h) in faces[i]:
##        # To draw a rectangle in a face
##                cv2.rectangle(f,(x,y),(x+w,y+h),(255,255,0),2)
##                #roi_gray = gray[y:y+h, x:x+w]
##                roi_color[i] = f[y:y+h, x:x+w]
##                count+=1
##                file=destination+'\\'+str(count)+'.jpg'
##                print(roi_color[i])
##                cv2.imwrite(file,roi_color[i])

            #print(faces[i])
            ##        gray = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)
            ret[i], filtered[i] = cv2.threshold(Rfilter[i], 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
            filtered[i] = cv2.medianBlur(f, 5)
            edged[i] = cv2.Canny(gray[i], 50, 50)
            fg_mask[i] = bg_subtractor.apply(filtered[i], None, 0.01)
            cv2.imshow(window[i], gray[i])
            contours, hierarchy = cv2.findContours(fg_mask[i], cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
            #cnt=0
            cntr = []

            for contour in contours:
                contourSize = cv2.contourArea(contour)
                if (contourSize > 10000):

                    cntr.append(contour)
                    cnt+=1
            #cnt=0
            #print(len(cntr))
            for c in range(0, len(cntr)):
                #print(cntr)
                #cntr = [None] * len[name]
                #cntr.append(ii)
                try:

                    (x[c], y[c], w[c], h[c]) = cv2.boundingRect(cntr[c])
                #print(cv2.boundingRect(cntr[c]))
                    cv2.rectangle(f, (x[c], y[c]), (x[c] + w[c] - 1, y[c] + h[c] - 1),(255, 255, 0), 1)
                    cropedframe[c]=f[y[c]:y[c]+h[c], x[c]:x[c]+w[c]]

                    filename=dst+'\\'+str(cnt)+'.jpg'
                    cv2.imwrite(filename,cropedframe[c])
                    # write the image
                    ret[c], enc[c] = cv2.imencode("*.bmp", cropedframe[c])
                    results = alpr.recognize_array(bytes(bytearray(enc[c])))
                    pos_frame = cap[i].get(cv2.CAP_PROP_POS_FRAMES)

                except (IndexError, TypeError) :
                    print("")

            for (x,y,w,h) in faces[i]:
                try:
        # To draw a rectangle in a face
                    cv2.rectangle(f,(x,y),(x+w,y+h),(255,255,0),2)
                #roi_gray = gray[y:y+h, x:x+w]
                    roi_color[i] = f[y:y+h, x:x+w]
                    count+=1
                    file=destination+'\\'+str(count)+'.jpg'

                    cv2.imwrite(file,roi_color[i])

                except (IndexError, TypeError) :

                    if results['results']:

                        print('License plate Detected and Recorded')

                        datentime = time.asctime( time.localtime(time.time()) )
                #time =  datetime.datetime.now().time()
                #date = datetime.datetime.now().date()
                        plateno = results['results'][0]['plate']
                        confidance=results['results'][0]['confidence']
                        #window=results['results'][0]['window']

                        print('Plate: ',plateno, 'Confidance: ',confidance,'Window: ',window[i],'Frame_no:',str(pos_frame),)
                        print("\n")

                        newFileWriter.writerow([plateno,cnt, filename, datentime, confidance,window[i],pos_frame]) # write the csv file
                        c=0
                        cntr=0

    if cv2.waitKey(3) ==27:
       break
    if cap[i].get(cv2.CAP_PROP_POS_FRAMES) == cap[i].get(cv2.CAP_PROP_FRAME_COUNT):
                        # If the number of captured frames is equal to the total number of frames,
                        # we stop
                        break

#print("cuurent running",window[0])
#print("cuurent running",window[1])
for c in cap:
    if c is not None:
        c.release()

cv2.destroyAllWindows()


import unittest
from dmLibrary import create_app
from dmLibrary.external.googleBook import GoogleBook
import time

import logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

class TestBookClass(unittest.TestCase):
    def setUp(self):
        app = create_app()
        self.ctx = app.app_context()
        self.ctx.push()
        self.client = app.test_client()
        self.gb = GoogleBook()
        logger.debug('logged from test_something')

    def tearDown(self):
        """Do the testing """
        pass

    def test_postBook(self):
        """
        test post new book, try update, and delete it
        """

        data = {
            "title":"เรียนรู้วิถีชีวิต ประเพณี พิธีกรรม และความเชื่อแบบบูรณาการ",
            "isbn":"9748846709"
        }
        response = self.client.post('/api/v1/books',json=data)
        logger.info(f"rawResp: {response.data}")
        self.assertEqual(response.status_code, 201)
        respData = response.get_json()
        self.assertEqual(respData["data"]['title'], data['title'])
        self.assertEqual(respData["data"]['isbn'], data['isbn'])
        pageCount = self.gb.getPageCount(data["isbn"])
        self.assertEqual(respData["data"]['pageCount'], 159)
        logger.info(f"respData: {respData}")
        book_id = respData["data"]["id"]

        # try update the title
        data = {
            "title":"test update Book"
        }
        response = self.client.patch(f'/api/v1/books/{book_id}',json=data)
        logger.info(f"rawResp: {response.data}")
        self.assertEqual(response.status_code, 200)
        respData = response.get_json()
        self.assertEqual(respData["data"]['title'], data['title'])
        # delete it
        response = self.client.delete(f'/api/v1/books/{book_id}')
        logger.info(f"rawResp: {response.data}")
        self.assertEqual(response.status_code, 200)


import numpy as np
import pandas as pd
import re
from itertools import chain
from IPython.display import Markdown, display
from more_itertools import unique_everseen

from src.davy_data_cleaning import *
from src.manasa_data_cleaning import *
from src.naomi_data_cleaning import *
from src.sara_data_cleaning import *

if __name__ == "__main__":
    #Reading the merged raw dataset from
    data = pd.read_csv('https://raw.githubusercontent.com/FrancescoMariottini/project3/main/inputs/all_sales_data.csv',sep=',',dtype='unicode')

    data['hyperlink'] = clean_hyperlink_column(data)
    data["postcode"] = postcodes_fun(data)
    data["house_is"] = house_is(data)
    data["property_subtype"] = process_property_types(data)
    data["price"] = clean_price_column(data)
    data["sale"] = clean_sale_column(data)
    data["rooms_number"] = process_num_rooms_col(data)
    data["area"] = process_area_col(data)
    data["kitchen_has"] = bool_to_bin("kitchen_has", int, data)
    data["furnished"] = bool_to_bin("furnished", int, data)
    data["open_fire"] = process_open_fire_col(data)
    data["terrace"] = process_terrace(data)
    data["terrace_area"] = process_terrace_area(data)
    data["garden"] = clean_garden_column(data)
    data["garden_area"] = clean_garden_area(data)
    data["land_surface"] = process_land_surf_col(data)
    data["land_plot_surface"] = process_land_plot_col(data)
    data["facades_number"] = facades_num(data)
    data["swimming_pool_has"] = clean_swimming_pool(data)
    data["building_state"] = process_building_state(data)

    #Drop duplicates
    data_dupes = data
    data_dupes = data_dupes.drop('source', axis=1)
    data_dupes = data_dupes.drop('hyperlink', axis=1)
    data_dupes = data_dupes.drop_duplicates()

    #Save the two final files
    data_dupes.to_csv('Datasets/final_with_dupes.csv')
    with open('Datasets/final_with_dupes.csv','r') as f, open('Datasets/final_no_dupes.csv','w') as out_file:
        out_file.writelines(unique_everseen(f))


#!/usr/bin/env python
# coding: utf-8

import math
from tqdm.auto import tqdm
import pandas as pd
import ujson
import numpy as np;
import pickle
import os
import xnetwork as xn
import bgzf
import json
import struct
import os
import igraph as ig
import matplotlib.pyplot as plt
from os.path import join as PJ
import gzip
import sys
import re
from datetime import datetime as dt
import sys
import matplotlib
import nltk

nltk.download("stopwords")
from nltk.corpus import stopwords

matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['ps.fonttype'] = 42

MAGPath = PJ("..","Data","MAG") #PATH TO THE MAG DATA
MAGPath = PJ("/gpfs/sciencegenome/AuthorDynamics/MAG/mag-2021-01-05/mag")
dataPath = PJ("..","Data","Processed")
figuresPath = PJ("..","Figures")
graycolor = "#808080"

HCAProjectDataPath = PJ("..","Data","Publications","Biomedical Papers","HumanCellAtlas.json")
HGPTitles2MAGPath = PJ("..","Data","Publications","Biomedical Papers","HGP_Publications_MAG.tsv")

minPaperCount = 1;
rebuildAll = False

temporaryPath = PJ("..","Temp")
os.makedirs(dataPath, exist_ok=True)
os.makedirs(temporaryPath, exist_ok=True)

def dictFromList(aList):
    return {aList[i]:i for i in range(len(aList))}

STOPWORDS = stopwords.words('english')
STOPWORDS = set(STOPWORDS)
import re
def text_prepare(text, STOPWORDS=STOPWORDS):
    """
        text: a string

        return: a clean string
    """
    REPLACE_BY_SPACE_RE = re.compile('[\n\"\'/(){}\[\]\|@,;#\.]')
    text = re.sub(REPLACE_BY_SPACE_RE, ' ', text)
    text = re.sub(' +', ' ', text)
    text = text.lower()

    # delete stopwords from text
    text = ' '.join([word for word in text.split() if word not in STOPWORDS])
    text = text.strip()
    return text

sortedPapersFilePath = PJ(dataPath,"MAGPapersByPaper.txt");
bgzPapersFilePath = PJ(dataPath,"MAGPapers.bgz");

papersDataTypes = [
    ("PaperId", int),
    ("Rank", int),
    ("Doi", str),
    ("DocType", str),
    ("PaperTitle", str),
    ("OriginalTitle", str),
    ("BookTitle", str),
    ("Year", int),
    ("Date", str),
    ("OnlineDate", str),
    ("Publisher", str),
    ("JournalId", int),
    ("ConferenceSeriesId", int),
    ("ConferenceInstanceId", int),
    ("Volume", str),
    ("Issue", str),
    ("FirstPage", str),
    ("LastPage", str),
    ("ReferenceCount", int),
    ("CitationCount", int),
    ("EstimatedCitation", int),
    ("OriginalVenue", str),
    ("FamilyId", int),
    ("FamilyRank",int),
    ("CreatedDate", str),
]

# Getting paper Metadata
def processTitleYearJournal(bgzPapersFilePath,papersDataTypes):
    estimatedCount = 233745561;
    index2magID = []
    index2Title = []
    index2DOI = []
    index2Year = []
    index2OnlineDate = []
    index2JournalID = []
    count = 0;
    with bgzf.open(bgzPapersFilePath,"rb") as infd:
        pbar = tqdm(total=estimatedCount);
        while True:
            pbar.update(1);
            lengthData = infd.read(8);
            if(len(lengthData)==8):
                length = struct.unpack("<q",lengthData)[0];
            else:
                break;
            data = infd.read(length);
            paperIndex = struct.unpack("<q",data[0:8])[0];
            currentPointer = 8;
    #         paperID = struct.unpack("<q",data[8:16])[0];
            dataDict = {};
            for (typeIndex,(typeName,typeType)) in enumerate(papersDataTypes):
                if(typeType==int):
                    entryValue, = struct.unpack("<q",data[currentPointer:currentPointer+8]);
                    currentPointer+=8;
                    dataDict[typeName] = entryValue;
                else:
                    entryLength, = struct.unpack("<q",data[currentPointer:currentPointer+8]);
                    currentPointer+=8;
                    entryData = data[currentPointer:currentPointer+entryLength].decode("utf8");
                    currentPointer+=entryLength;
                    dataDict[typeName] = entryData;
            index2Title.append(dataDict["PaperTitle"])
            index2Year.append(dataDict["Year"])
            index2magID.append(dataDict["PaperId"])
            index2DOI.append(dataDict["Doi"])
            index2OnlineDate.append(dataDict["OnlineDate"])
            index2JournalID.append(dataDict["JournalId"])
            count+=1;
#             if(count>100000):
#                 break;
    return index2Title,index2magID,index2Year,index2OnlineDate,index2JournalID,index2DOI;

index2Title,index2magID,index2Year,index2OnlineDate,index2JournalID,index2DOI=processTitleYearJournal(bgzPapersFilePath,papersDataTypes)
index2FirstYear = []
for i in tqdm(range(len(index2Year))):
    year = index2Year[i]
    if(index2OnlineDate[i]):
        onlineDate = int(index2OnlineDate[i][0:4])
        if(onlineDate<year):
            year=onlineDate
    index2FirstYear.append(year)
del index2OnlineDate

magID2Index = dictFromList(index2magID)

# Loading extended attributes

# PAPER EXTENDED ATTRIBUTES
# Column #	Name	Type	Note
# 1	PaperId	long	FOREIGN KEY REFERENCES Papers.PaperId
# 2	AttributeType	int	1: PatentId
#                       2: PubMedId
#                       3: PmcId
#                       4: Alternative Title
# 3	AttributeValue	string

paperExtendedAttributesFilePath = PJ(MAGPath,"PaperExtendedAttributes.txt");
index2PubMedId = [None]*len(index2magID)
index2PmcId = [None]*len(index2magID)
missingPaperIDs = set()
estimatedCount = 233745561;
previousPaperID = 0
currentIndex = 0
magCount = len(index2magID)
with open(paperExtendedAttributesFilePath,"rt") as fd:
    pbar = tqdm(total=estimatedCount);
    pbar2 = tqdm(total=estimatedCount);
    for line in fd:
        pbar.update(1);
        entries = line.strip().split("\t");
        paperID = int(entries[0]);
        if(paperID<previousPaperID):
            print("Problem!!! Out of order!")
            break
        previousPaperID = paperID;
        currentMAGID = index2magID[currentIndex]
        while(currentMAGID<paperID and currentIndex<magCount):
            currentIndex+=1;
            currentMAGID = index2magID[currentIndex]
            pbar2.update(1);
        if(currentMAGID==paperID):
            attributeType = int(entries[1]);
            attributeValue = str(entries[2]);
            if(attributeType==2):
                index2PubMedId[currentIndex] = attributeValue
            if(attributeType==3):
                index2PmcId[currentIndex] = attributeValue
        else:
            continue;

# Matching HCA Projects
with open(HCAProjectDataPath,"rt") as fd:
    HCAProjectData = ujson.load(fd);
for entry in HCAProjectData:
    if("links" in entry):
        entry.update({x.split(": ")[0]:x.split(": ")[1] for x in entry["links"] if len(x.split(": "))>1})

#By
#PMC
#DOI
#TITLES
#PUBMED

#HCA Match2MAG
HCAProjectDOISets = {}
HCAProjectTitleSets = {}
HCAProjectPMCSets = {}
HCAProjectPUBMEDSets = {}
for entry in HCAProjectData:
    if("DOI" in entry):
        HCAProjectDOISets[entry["DOI"].lower()] = entry;

    if("PMC" in entry):
        HCAProjectPMCSets[entry["PMC"].lower()] = entry;

    if("PUBMED" in entry):
        HCAProjectPUBMEDSets[entry["PUBMED"].lower()] = entry;

    HCAProjectTitleSets[text_prepare(entry["title"].lower())] = entry;

selectedDOIMatches = {}
selectedTitleMatches = {}
selectedPMCMatches = {}
selectedPUBMEDMatches = {}
for i in tqdm(range(len(index2Year))):
    DOI = index2DOI[i].lower()
    if(DOI in HCAProjectDOISets):
        selectedDOIMatches[DOI] = i;
        continue;

    PMCID = index2PmcId[i]
    if(PMCID):
        PMCID = PMCID.lower()
        if(PMCID in HCAProjectPMCSets):
            selectedPMCMatches[PMCID] = i;
            continue;

    PUBMEDID = index2PubMedId[i]
    if(PUBMEDID):
        PUBMEDID = PUBMEDID.lower()
        if(PUBMEDID in HCAProjectPUBMEDSets):
            selectedPUBMEDMatches[PUBMEDID] = i;
            continue;

    reducedTitle = text_prepare(index2Title[i].lower())
    if(reducedTitle in HCAProjectTitleSets):
        selectedTitleMatches[reducedTitle] = i;
        continue;

#Matching HCA projects

HCAProjectMAGIndices = set(selectedDOIMatches.values()).union(set(selectedTitleMatches.values()))

#Human Cell Atlas
HCAMissingData = []
for entry in HCAProjectData:
    matchesCount = 0;
    if("DOI" in entry and entry["DOI"].lower() in selectedDOIMatches):
        matchesCount+=1
    if("title" in entry and text_prepare(entry["title"].lower()) in selectedTitleMatches):
        matchesCount+=1
    if(matchesCount>0):
#         print(checkDouble);
        continue
    HCAMissingData.append(entry)
# print(missingData)

#Saving HCA Matched MAG Indices
with open(PJ("..","Data","Publications","Biomedical Papers","HCA_MAGIndices.json"),"wt") as fd:
    ujson.dump(list(HCAProjectMAGIndices),fd)

with open(PJ("..","Data","Publications","Biomedical Papers","HCA_MissingEntries.json"),"wt") as fd:
    ujson.dump(list(HCAMissingData),fd)

#Human Genome Project
dfHGP = pd.read_csv(HGPTitles2MAGPath,sep="\t")
HGPProjectMAGIndices = set()
HGPMissingData = []
for referece,url in dfHGP[["Reference","MAG URL"]].to_numpy():
    if(url==url):
        magID = int(re.search(r'https\:\/\/academic\.microsoft\.com\/paper\/(.*?)\/reference', url).group(1))
        if(magID in magID2Index):
            index = magID2Index[magID]
            HGPProjectMAGIndices.add(index)
            continue;

    HGPMissingData.append(referece);

with open(PJ("..","Data","Publications","Biomedical Papers","HGP_MAGIndices.json"),"wt") as fd:
    ujson.dump(list(HGPProjectMAGIndices),fd)

with open(PJ("..","Data","Publications","Biomedical Papers","HGP_MissingEntries.json"),"wt") as fd:
    ujson.dump(list(HGPMissingData),fd)

HBMapDataPaths = [
    PJ("..","Data","Publications","Biomedical Papers","Pubhl_Export_21Jan2021_084406.csv"),
    PJ("..","Data","Publications","Biomedical Papers","Pubhl_Export_21Jan2021_084445.csv"),
    PJ("..","Data","Publications","Biomedical Papers","Publ_18Jan2021_014100_86642982.csv")
]

HBMapTitleKey = "Title (Link to full-text in PubMed Central)"
HBMapDfs = [pd.read_csv(filename,dtype={"PMC ID":str,"PMID":str}) for filename in HBMapDataPaths]
HBMapDf = pd.concat(HBMapDfs,sort=False)

HBMapTitles = HBMapDf[HBMapTitleKey].to_list()
HBMapReducedTitles = set(text_prepare(title.lower()) for title in HBMapTitles if isinstance(title,str))
HBMapPMCIDs = set(pmcid.lower() for pmcid in HBMapDf["PMC ID"].to_list() if isinstance(pmcid,str))
HBMapPUBMEDIDs = set(pubmedID.lower() for pubmedID in HBMapDf["PMID"].to_list() if isinstance(pubmedID,str))

HBMapSelectedDOIMatches = {}
HBMapSelectedTitleMatches = {}
HBMapSelectedPMCMatches = {}
HBMapSelectedPUBMEDMatches = {}

pbarnew = tqdm(desc='Added');
for i in tqdm(range(len(index2Year))):
    PMCID = index2PmcId[i]
    if(PMCID):
        PMCID = PMCID.lower()
        if(PMCID in HBMapPMCIDs):
            HBMapSelectedPMCMatches[PMCID] = i;
            pbarnew.update(1);
            continue;

    PUBMEDID = index2PubMedId[i]
    if(PUBMEDID):
        PUBMEDID = PUBMEDID.lower()
        if(PUBMEDID in HBMapPUBMEDIDs):
            HBMapSelectedPUBMEDMatches[PUBMEDID] = i;
            pbarnew.update(1);
            continue;

    reducedTitle = text_prepare(index2Title[i].lower())
    if(reducedTitle in HBMapReducedTitles):
        HBMapSelectedTitleMatches[reducedTitle] = i;
        pbar2.update(1);
        continue;

HBMapProjectMAGIndices = set(HBMapSelectedPMCMatches.values()).union(set(HBMapSelectedPUBMEDMatches.values()).union(set(HBMapSelectedTitleMatches.values())))

HBMapMissingData = []
for title,PMCID,PMID in HBMapDf[[HBMapTitleKey,"PMC ID","PMID"]].to_numpy():
    matchesCount = 0;
    if(PMCID in HBMapSelectedPMCMatches):
        matchesCount+=1
    if(PMID in HBMapSelectedPUBMEDMatches):
        matchesCount+=1
    if(title==title and text_prepare(title.lower()) in HBMapSelectedTitleMatches):
        matchesCount+=1
    if(matchesCount>0):
#         print(checkDouble);
        continue
    if(title==title):
        HBMapMissingData.append(title)

HBMapMissingData = set(HBMapMissingData)

#Included by manually searching missing in MAG
HBMapManualMAGIDIncludes = [
    3092960214,
    3099559583,
    3099294714,
    3089472526,
    3100967665,
    3038602829,
    3092108146,
    3081828282,
    3099559583,
]

#Only adding matches in currentMAGData
for magID in HBMapManualMAGIDIncludes:
    if(magID in magID2Index):
        HBMapProjectMAGIndices.add(magID2Index[magID])

with open(PJ("..","Data","Publications","Biomedical Papers","HBMap_MAGIndices.json"),"wt") as fd:
    ujson.dump(list(HBMapProjectMAGIndices),fd)

with open(PJ("..","Data","Publications","Biomedical Papers","HBMap_MissingEntries.json"),"wt") as fd:
    ujson.dump(list(HBMapMissingData),fd)

# In[402]:

#Statistics:
print("Summary")
print("HCA")
print("\tMatches: %d\n\tMissing:%d"%(len(HCAProjectMAGIndices),len(HCAMissingData)))
print("HuBMAP")
print("\tMatches: %d\n\tMissing:%d"%(len(HBMapProjectMAGIndices),len(HBMapMissingData)))
print("HGP")
print("\tMatches: %d\n\tMissing:%d"%(len(HGPProjectMAGIndices),len(HGPMissingData)))

# Testing open
with open(PJ("..","Data","Publications","Biomedical Papers","HCA_MAGIndices.json"),"rt") as fd:
    HCAProjectMAGIndices = set(ujson.load(fd))

with open(PJ("..","Data","Publications","Biomedical Papers","HBMap_MAGIndices.json"),"rt") as fd:
    HBMapProjectMAGIndices = set(ujson.load(fd))

with open(PJ("..","Data","Publications","Biomedical Papers","HGP_MAGIndices.json"),"rt") as fd:
    HGPProjectMAGIndices = set(ujson.load(fd))

# Recovering References from MAG

#references FromTo
estimatedCount = 79684807
bgzReferencesFromToPath = PJ(dataPath,"MAGPaperReferencesFromTo.bgz")
entriesCount = 0
# setOfJournalsReferenced = set()
# setOfJournalsWithPublications = set()
HCACitationsPerYear = {}
HBMapCitationsPerYear = {}
HCAHuBMapCitationsPerYear = {}
HGPCitationsPerYear = {}

projectCitations = {
    "HCA":(HCAProjectMAGIndices,HCACitationsPerYear),
    "HuBMAP":(HBMapProjectMAGIndices,HBMapCitationsPerYear),
    "HuBMAP+HCA":(HBMapProjectMAGIndices.union(HCAProjectMAGIndices),HCAHuBMapCitationsPerYear),
    "HGP":(HGPProjectMAGIndices,HGPCitationsPerYear)
}

MAGID2CitationYear = {}
with bgzf.open(bgzReferencesFromToPath,"rb") as infd:
    pbar = tqdm(total=estimatedCount)
    count = 0
    while True:
        pbar.update(1)
        lengthData = infd.read(8)
        if(len(lengthData)==8):
            length = struct.unpack("<q",lengthData)[0]
        else:
            break
        data = infd.read(length)
        edgesCount = length/8-1
        fmt = "<%dq" % (edgesCount+1)
        edgeData  = list(struct.unpack(fmt,data))
        fromIndex = edgeData[0]
        fromID = index2magID[fromIndex]
        count+=1
        if(len(edgeData)>1):
            year = index2FirstYear[fromIndex]
            for projectName,(indices,citationsPerYear) in projectCitations.items():
                toIndices = set(edgeData[1:]).intersection(indices)
                if(toIndices):
                    if(year not in citationsPerYear):
                        citationsPerYear[year] = set()
                    citationsPerYear[year].add(fromIndex)
                    for toID in [index2magID[index] for index in toIndices]:
                        if(toID not in MAGID2CitationYear):
                            MAGID2CitationYear[toID] = []
                        MAGID2CitationYear[toID].append((fromID,year))

HCAReferencesPerYear = {}
HBMapReferencesPerYear = {}
HCAHuBMapReferencesPerYear = {}
HGPReferencesPerYear = {}

projectReferences = {
    "HCA":(HCAProjectMAGIndices,HCAReferencesPerYear),
    "HuBMAP":(HBMapProjectMAGIndices,HBMapReferencesPerYear),
    "HuBMAP+HCA":(HBMapProjectMAGIndices.union(HCAProjectMAGIndices),HCAHuBMapReferencesPerYear),
    "HGP":(HGPProjectMAGIndices,HGPReferencesPerYear)
}

bgzReferencesToFromPath = PJ(dataPath,"MAGPaperReferencesToFrom.bgz");
entriesCount = 0;
MAGID2ReferenceYear = {};
with bgzf.open(bgzReferencesToFromPath,"rb") as infd:
    pbar = tqdm(total=estimatedCount);
    count = 0;
    while True:
        pbar.update(1);
        lengthData = infd.read(8);
        if(len(lengthData)==8):
            length = struct.unpack("<q",lengthData)[0];
        else:
            break;
        data = infd.read(length);
        edgesCount = length/8-1;
        fmt = "<%dq" % (edgesCount+1)
        edgeData  = list(struct.unpack(fmt,data))
        fromIndex = edgeData[0];
        fromID = index2magID[fromIndex]
        count+=1;
        if(len(edgeData)>1):
            for projectName,(indices,referencesPerYear) in projectReferences.items():
                toIndices = set(edgeData[1:]).intersection(indices)
                if(toIndices):
                    if(year not in referencesPerYear):
                        referencesPerYear[year] = set();
                    referencesPerYear[year].add(fromIndex);
                    for toIndex,toID in [(index,index2magID[index]) for index in toIndices]:
                        year = index2FirstYear[fromIndex];
                        if(toID not in MAGID2ReferenceYear):
                            MAGID2ReferenceYear[toID] = [];
                        MAGID2ReferenceYear[toID].append((fromID,year));

projectCitationsIndices = {}
projectReferencesIndices = {}

projectCitationsEdges = {}
projectReferencesEdges = {}

for projectName,(indices,citationsPerYear) in projectCitations.items():
    citationsEdges = []
    citationsSet = set()
    projectCitationsEdges[projectName] = citationsEdges
    projectCitationsIndices[projectName] = citationsSet
    for paperIndex in indices:
        paperID = index2magID[paperIndex]
        if paperID in MAGID2CitationYear:
            for toID,year in MAGID2CitationYear[paperID]:
                citationsEdges.append((paperID,toID))
                citationsSet.add(toID)

for projectName,(indices,referencesPerYear) in projectReferences.items():
    referencesEdges = []
    referencesSet = set()
    projectReferencesEdges[projectName] = referencesEdges
    projectReferencesIndices[projectName] = referencesSet
    for paperIndex in indices:
        paperID = index2magID[paperIndex]
        if paperID in MAGID2ReferenceYear:
            for fromID,year in MAGID2ReferenceYear[paperID]:
                referencesEdges.append((paperID,fromID))
                referencesSet.add(fromID)

HCAAffiliationsPerYear = {};
HuBMapAffiliationsPerYear = {};
HGPAffiliationsPerYear = {};

HCAAuthorsPerYear = {};
HuBMapAuthorsPerYear = {};
HGPAuthorsPerYear = {};

projectAuthorAffiliations = {
    "HCA":(HCAProjectMAGIndices,HCAAffiliationsPerYear,HCAAuthorsPerYear),
    "HuBMAP":(HBMapProjectMAGIndices,HuBMapAffiliationsPerYear,HuBMapAuthorsPerYear),
    "HuBMAP+HCA":(set(HBMapProjectMAGIndices).union(set(HCAProjectMAGIndices)),HuBMapAffiliationsPerYear,HuBMapAuthorsPerYear),
    "HGP":(HGPProjectMAGIndices,HGPAffiliationsPerYear,HGPAuthorsPerYear)
}

allMAGIDs = {index2magID[entry] for entry in HCAProjectMAGIndices|HBMapProjectMAGIndices|HGPProjectMAGIndices}

MAGID2AuthorsIDs = {}
MAGID2AffiliationIDs = {}

estimatedCount = 655732564;# Used just to display status
with open(PJ(MAGPath,"PaperAuthorAffiliations.txt"),"rt") as fd:
    pbar = tqdm(total=estimatedCount);
    for line in fd:
        pbar.update(1);
        entries = line.split("\t");
        paperID = int(entries[0]);
        if(paperID in allMAGIDs):
            paperAuthorID = int(entries[1]);
            if(entries[2].isdigit()):
                paperAffiliationID = int(entries[2]);
            else:
                paperAffiliationID=0;
            authorOrder = int(entries[3]);
            paperIndex = magID2Index[paperID]
            paperYear = index2FirstYear[paperIndex]
            if(paperID not in MAGID2AuthorsIDs):
                MAGID2AuthorsIDs[paperID] = [];

            if(paperID not in MAGID2AffiliationIDs):
                MAGID2AffiliationIDs[paperID] = [];

            MAGID2AuthorsIDs[paperID].append(paperAuthorID);
            MAGID2AffiliationIDs[paperID].append(paperAffiliationID);

            for projectName,(indices,affiliationsPerYear,authorsPerYear) in projectAuthorAffiliations.items():
                if(paperIndex in indices):
                    if(paperYear not in authorsPerYear):
                        authorsPerYear[paperYear] = set();
                    authorsPerYear[paperYear].add(paperAuthorID);
                    if(paperAffiliationID):
                        if(paperYear not in affiliationsPerYear):
                            affiliationsPerYear[paperYear] = set();
                        affiliationsPerYear[paperYear].add(paperAffiliationID);

indicesByProject = {
"HCA" : HCAProjectMAGIndices,
"HuBMAP" : HBMapProjectMAGIndices,
"HuBMAP+HCA" : set(HBMapProjectMAGIndices).union(set(HCAProjectMAGIndices)),
"HGP" : HGPProjectMAGIndices
}

magIDsByProject = {}
allMAGIDs = set()
for projectName,indices in indicesByProject.items():
    magIDsByProject[projectName] = {index2magID[index] for index in indices}
    allMAGIDs.update(magIDsByProject[projectName])

exportFilesByProject = {}
for projectName,indices in indicesByProject.items():
    exportFilesByProject[projectName] = open(PJ("..","Data", "Publications","Biomedical_Papers_%s.tsv"%projectName),"w")
    headers = [entryName for entryName,_ in papersDataTypes]
    headers.append("Author IDs")
    headers.append("Affiliation IDs")
    headers.append("Citation IDs")
    headers.append("Citation Years")
    exportFilesByProject[projectName].write("\t".join(headers));
    exportFilesByProject[projectName].write("\n");

with open(PJ(MAGPath,"Papers.txt"),"rt") as fd:
    pbar = tqdm(total=estimatedCount);
    pbar2 = tqdm(desc="matches");
    for line in fd:
        pbar.update(1);
        entries = line.strip().split("\t");
        paperID = int(entries[0]);
        if(paperID in allMAGIDs):
            pbar2.update(1)
            for projectName,magIDs in magIDsByProject.items():
                if(paperID in magIDs):
                    exportFilesByProject[projectName].write(line.strip());
                    exportFilesByProject[projectName].write("\t%s"%";".join([str(entry) for entry in MAGID2AuthorsIDs[paperID] if entry!=0]))
                    exportFilesByProject[projectName].write("\t%s"%";".join([str(entry) for entry in MAGID2AffiliationIDs[paperID] if entry!=0]))
                    if(paperID in MAGID2CitationYear):
                        exportFilesByProject[projectName].write("\t%s"%";".join([str(entry) for entry,year in MAGID2CitationYear[paperID] if entry!=0]))
                        exportFilesByProject[projectName].write("\t%s"%";".join([str(year) for entry,year in MAGID2CitationYear[paperID] if year!=0]))
                    else:
                        exportFilesByProject[projectName].write("\t");
                        exportFilesByProject[projectName].write("\t");
                    exportFilesByProject[projectName].write("\n")
for projectName,indices in indicesByProject.items():
    exportFilesByProject[projectName].close()

# Loading MAG Data
journalIndex2MAGID = np.loadtxt(PJ(dataPath,"MAGJournalIndex2ID.txt.gz"),dtype=int)
journalMAGID2Index = dictFromList(journalIndex2MAGID)
journalMAGID2Index = {ID:index for index,ID in enumerate(journalIndex2MAGID)};
with gzip.open(PJ(dataPath,"MAGJournalNames.txt.gz"),"rt") as fd:
    journalMAGNames = [line.strip() for line in fd]

with gzip.open(PJ(dataPath,"MAGJournalLongNames.txt.gz"),"rt") as fd:
    journalMAGLongNames = [line.strip() for line in fd]

with gzip.open(PJ(dataPath,"MAGJournalRankings.txt.gz"),"rt") as fd:
    journalMAGRankings = [int(line.strip()) for line in fd]

with gzip.open(PJ(dataPath,"MAGJournalPublishers.txt.gz"),"rt") as fd:
    journalMAGPublishers = [line.strip() for line in fd]

with gzip.open(PJ(dataPath,"MAGJournalPaperCounts.txt.gz"),"rt") as fd:
    journalMAGPaperCounts = [int(line.strip()) for line in fd]

with gzip.open(PJ(dataPath,"MAGJournalISSN.txt.gz"),"rt") as fd:
    journalMAGISSN = [line.strip() for line in fd]

dfBioMed = {
    "HuBMAP+HCA": pd.read_csv(PJ("..","Data","Publications","Biomedical_Papers_HuBMAP+HCA.tsv"),sep="\t"),
    "HGP": pd.read_csv(PJ("..","Data","Publications","Biomedical_Papers_HGP.tsv"),sep="\t")
}

#Fixing journal names
for projectName,df in dfBioMed.items():
    df["Journal Name"] = [journalMAGNames[journalMAGID2Index[int(ID)]] if ID else "NA" for ID in df["JournalId"]]
    df["Journal Long Name"] = [journalMAGLongNames[journalMAGID2Index[int(ID)]] if ID else "NA" for ID in df["JournalId"]]
    df.to_csv(PJ("..","Data", "Publications","Biomedical_Papers_%s.csv"%projectName))

import re
def canonizeName(name):
  name = name.strip("\"").strip("\'")
  name = name.lower().replace("."," ").replace("  "," ").replace("  "," ").strip()
  name = re.sub('^nat ','nature ',name)
  name = re.sub('journal ','j ',name)
  names = name.split()
  return " ".join([subname[0:3] if len(subname)==4 else subname  for subname in names])

# Loading dictionaries for Journal matching
with open("../Data/ScienceMapJournalDictionary_Map2ID.json","rt") as fd:
    allToMapID = ujson.load(fd)

with open("../Data/ScienceMapJournalDictionary_Map2Name.json","rt") as fd:
    allToMapName = ujson.load(fd)

with open("../Data/ScienceMapJournalDictionary_ISSN2ID.json","rt") as fd:
    ISSNToMapID = ujson.load(fd)

from collections import Counter
noMatch = Counter()
def venueName2MapName(origName):
  name = canonizeName(origName)
  if(name in allToMapName):
    return allToMapName[name]
  noMatch.update([origName])
  return "NA"

def venueName2MapID(origName):
  name = canonizeName(origName)
  if(name in allToMapID):
    return allToMapID[name]
  noMatch.update([origName])
  return "NA"

def venueISSN2MapID(origISSN):
  name = origISSN.replace("-","").lower()
  if(origISSN in ISSNToMapID):
    return ISSNToMapID[origISSN]
  return "NA"

def getVenueData(paperID):
    journalID = index2JournalID[magID2Index[paperID]]
    if(journalID<0):
        return "NA","NA"
    journalIndex = journalMAGID2Index[journalID]
    longName = journalMAGLongNames[journalIndex]
    name = journalMAGNames[journalIndex]
    issn = journalMAGISSN[journalIndex]
    mapID = venueISSN2MapID(issn)
    if(mapID=="NA"):
        mapID = venueName2MapID(issn)
    if(mapID=="NA"):
        mapID = venueName2MapID(name)
    if(mapID=="NA"):
        mapID = venueName2MapID(longName)
    return name,mapID

def getYearData(paperID):
    return index2Year[magID2Index[paperID]]

minYear = 1980
maxYear = 2020

# Saving all publication data
for projectName, edgeEntries in tqdm(projectCitationsEdges.items(), desc="Project"):
  edgesData = {
      "Citing ID": [],
      "Citing Venue Name": [],
      "Citing Venue SciMap ID": [],
      "Citing Publication Year": [],
      "Cited ID": [],
      "Cited Venue Name": [],
      "Cited Venue SciMap ID": [],
      "Cited Publication Year": [],
  }
  for (citedEntryID,citingEntryID) in tqdm(edgeEntries):
    citedEntryYear = 0
    citedEntryVenueName,citedEntryVenueMapID = getVenueData(citedEntryID)
    citedEntryYear = getYearData(citedEntryID)
    if(citedEntryYear<minYear or citedEntryYear>maxYear):
      continue
    #Received Citations
    citingEntryVenueName,citingEntryVenueMapID = getVenueData(citingEntryID)
    citingYear = getYearData(citingEntryID)
    if(citingYear<minYear or citingYear>maxYear):
        continue
    edgesData["Citing ID"].append(citingEntryID)
    edgesData["Citing Venue Name"].append(citingEntryVenueName)
    edgesData["Citing Venue SciMap ID"].append(citingEntryVenueMapID)
    edgesData["Citing Publication Year"].append(citingYear)
    edgesData["Cited ID"].append(citedEntryID)
    edgesData["Cited Venue Name"].append(citedEntryVenueName)
    edgesData["Cited Venue SciMap ID"].append(citedEntryVenueMapID)
    edgesData["Cited Publication Year"].append(citedEntryYear)
  dfEdges = pd.DataFrame.from_dict(edgesData)
  dfEdges.to_csv(PJ("..","Data","ToMapOfScience","Biomedical_Edges_Citing_%s_tomap.csv.gz"%(projectName)),index=False)

# Saving all publication data
for projectName, edgeEntries in tqdm(projectReferencesEdges.items(), desc="Project"):
  edgesData = {
      "Citing ID": [],
      "Citing Venue Name": [],
      "Citing Venue SciMap ID": [],
      "Citing Publication Year": [],
      "Cited ID": [],
      "Cited Venue Name": [],
      "Cited Venue SciMap ID": [],
      "Cited Publication Year": [],
  }
  for (citingEntryID,citedEntryID) in tqdm(edgeEntries):
    citedEntryYear = 0
    citedEntryVenueName,citedEntryVenueMapID = getVenueData(citedEntryID)
    citedEntryYear = getYearData(citedEntryID)
    if(citedEntryYear<minYear or citedEntryYear>maxYear):
      continue
    #Received Citations
    citingEntryVenueName,citingEntryVenueMapID = getVenueData(citingEntryID)
    citingYear = getYearData(citingEntryID)
    if(citingYear<minYear or citingYear>maxYear):
        continue
    edgesData["Citing ID"].append(citingEntryID)
    edgesData["Citing Venue Name"].append(citingEntryVenueName)
    edgesData["Citing Venue SciMap ID"].append(citingEntryVenueMapID)
    edgesData["Citing Publication Year"].append(citingYear)
    edgesData["Cited ID"].append(citedEntryID)
    edgesData["Cited Venue Name"].append(citedEntryVenueName)
    edgesData["Cited Venue SciMap ID"].append(citedEntryVenueMapID)
    edgesData["Cited Publication Year"].append(citedEntryYear)
  dfEdges = pd.DataFrame.from_dict(edgesData)
  dfEdges.to_csv(PJ("..","Data","ToMapOfScience","Biomedical_Edges_Referenced_%s_tomap.csv.gz"%(projectName)),index=False)

allProjectsIDs = set()
for projectName in ["HGP","HuBMAP+HCA"]:
    dfCiting = pd.read_csv(PJ("..","Data","ToMapOfScience","Biomedical_Edges_Citing_%s_tomap.csv.gz"%(projectName)))
    dfReferences = pd.read_csv(PJ("..","Data","ToMapOfScience","Biomedical_Edges_Referenced_%s_tomap.csv.gz"%(projectName)))
    dfPapers = pd.read_csv(PJ("..","Data","ToMapOfScience","Biomedical_Papers_%s_tomap.csv"%(projectName)))
    allProjectsIDs.update(list(dfCiting["Citing ID"]))
    allProjectsIDs.update(list(dfCiting["Cited ID"]))
    allProjectsIDs.update(list(dfReferences["Citing ID"]))
    allProjectsIDs.update(list(dfReferences["Cited ID"]))
    allProjectsIDs.update(list(dfPapers['PaperId']))

allProjectsID2Title = {}
for magID in allProjectsIDs:
    title = index2Title[magID2Index[magID]]
    allProjectsID2Title[magID] = title

with open(PJ("..","Data","Biomedical_MAGID2Title.json"),"wt") as fd:
    json.dump(allProjectsID2Title,fd)


import time
import celery.result
import requests

from funboost import boost, BrokerEnum
from funboost.assist.celery_helper import CeleryHelper

@boost('tets_funboost_celery_queue31a5', broker_kind=BrokerEnum.CELERY, concurrent_num=10,is_print_detail_exception=False,
       broker_exclusive_config={'celery_app_config':
                                    {'task_default_rate_limit': '1/s', }}
       )
def fa(x, y):
    time.sleep(3)
    print(6666, x, y)
    requests.get('123')
    1/0
    return x + y

@boost('tets_funboost_celery_queue31b', broker_kind=BrokerEnum.CELERY, concurrent_num=10,
       broker_exclusive_config={'celery_app_config':
                                    {'task_default_rate_limit': '2/s', }}
       )
def fb(a, b):
    time.sleep(2)
    print(7777, a, b)
    return a - b

if __name__ == '__main__':
    fa.consume()
    # fb.consume()

    for i in range(1):
        r = fa.push(i, i + 1)  # type: celery.result.AsyncResult
        # print(type(r), r)
        # print(r.get())
        # fb.push(i, i * 2)

    CeleryHelper.realy_start_celery_worker()

'''

linux celery 报错EntryPoints‘ object has no attribute ‘get‘ ，这么做
https://blog.csdn.net/weixin_73672704/article/details/129477312

pip install frozenlist==1.3.1 geopy==2.2.0 humanize==4.3.0 idna==3.3 importlib-metadata==4.12.0 jsonschema==4.9.0 korean_lunar_calendar==0.2.1 marshmallow==3.17.0 pyOpenSSL==22.0.0 pyrsistent==0.18.1 python-dotenv==0.20.0 pytz==2022.2.1 selenium==4.4.0 simplejson==3.17.6 sniffio==1.2.0 trio==0.21.0 urllib3==1.26.11 wsproto==1.1.0 zipp==3.8.1

# 如果上述安装仍未解决, 则执行以下安装(笔者执行完上述安装即能解决问题)
pip install backoff==2.1.2 colorama==0.4.5 croniter==1.3.5 cryptography==37.0.4 email-validator==1.2.1 flask-compress==1.12 flask-migrate==3.1.0 aiohttp==3.8.1 aiosignal==1.2.0 Mako==1.2.1 Babel==2.10.3

'''


from django import template
from staff.models import Staff
register = template.Library()

@register.simple_tag
def get(value):

    try:
        name = Staff.objects.filter(company=value)
    except Staff.DoesNotExist:
        name = 'vacant'

    return name

# register.filter('get', get)


import pandas as pd
from pyproj import Transformer
import matplotlib.pyplot as plt
import numpy as np
from typing import Tuple, Set, List

EARTH_RADIUS = 6378137.0
EARTH_CIRC_M = 2 * np.pi * EARTH_RADIUS
TILE_SIZE_PX = 256
METERS_PER_PIXEL_L0 = EARTH_CIRC_M / TILE_SIZE_PX
PYPROJ = Transformer.from_crs("EPSG:4326", "EPSG:3857")  # EPSG:900913 == EPSG:3857

# inspired from
# - https://www.maptiler.com/google-maps-coordinates-tile-bounds-projection/#5/150.51/68.42

def load_coords(filepath: str):
    """ Loads csv file with coordinates, and returns a numpy array with col-wise coordinates. """
    df = pd.read_csv(filepath, sep=',')
    coords_ll = df[['longitude', 'latitude']].to_numpy()
    coords_ll = coords_ll.transpose()
    return coords_ll

def ll_to_meters(coords_ll):
    # Converts given lat/lon in WGS84 Datum to XY in Spherical Mercator EPSG:3857 (aka EPSG:900913)
    coords_m = PYPROJ.transform(coords_ll[1], coords_ll[0])
    coords_m = np.array(coords_m)
    return coords_m

def meters_to_pixels(coords_m, level):
    # Converts EPSG:3857 to pyramid pixel coordinates in given zoom level
    coords_px = (coords_m + (EARTH_CIRC_M / 2.0)) / meters_per_pixel(level)
    return coords_px

def pixels_to_meters(coords_px, level):
    # Converts pixel coordinates in given zoom level to EPSG:3857
    coords_m = (coords_px * meters_per_pixel(level)) - (EARTH_CIRC_M / 2)
    return coords_m

def pixel_to_raster(coords_px, level):
    # Move the origin of pixel coordinates to top-left corner (like image)
    coords_rx = np.empty_like(coords_px)
    coords_rx[0] = coords_px[0]
    coords_rx[1] = map_size_px(level) - coords_px[1]
    return coords_rx

def raster_to_tile(coords_rx, tile_coord: Tuple[int, int]):
    # Move the origin of pixel coordinates to top-left corner (like image)
    # filter out points outside the tile
    bounds_rx = tile_bounds_rx(tile_coord)
    coords_ti = coords_rx - bounds_rx[:, 0:1]
    return coords_ti

def keep_in_tile_rx(coords_rx, tile_coord: Tuple[int, int]):
    bounds_rx = tile_bounds_rx(tile_coord)
    in_tile = np.vstack([
        coords_rx > bounds_rx[:, 0:1],
        coords_rx < bounds_rx[:, 2:3],
    ])
    in_tile = np.all(in_tile, axis=0)
    return coords_rx[:, in_tile]

def ll_to_raster(coords_ll, level):
    coords_m = ll_to_meters(coords_ll)
    coords_px = meters_to_pixels(coords_m, level=level)
    coords_rx = pixel_to_raster(coords_px, level=level)
    return coords_rx

def ll_to_tile(coords_ll, tile_coord: Tuple[int, int], level):
    coords_m = ll_to_meters(coords_ll)
    coords_px = meters_to_pixels(coords_m, level=level)
    coords_rx = pixel_to_raster(coords_px, level=level)
    coords_ti = raster_to_tile(coords_rx, tile_coord)
    return coords_ti

def nb_tiles(level: int):
    return 2 ** level

def get_tile_coord_list(level: int) -> List[Tuple[int, int]]:
    xx, yy = np.meshgrid(np.arange(nb_tiles(level)), np.arange(nb_tiles(level)))
    tile_coords = np.column_stack([xx.flatten(), yy.flatten()])
    tile_coords = [
        (x, y) for x, y in tile_coords
    ]
    return tile_coords

def map_size_px(level):
    return TILE_SIZE_PX << level

def tile_bounds_rx(tile_coords: Tuple[int, int]):
    # Returns bounds of the given tile in EPSG:900913 coordinates
    coords_tile = np.array([[tile_coords[0], tile_coords[1]]]).transpose()
    square = np.array([
        [0, 0],
        [1, 0],
        [1, 1],
        [0, 1]
    ]).transpose()

    coord_bounds_px = (square + coords_tile) * TILE_SIZE_PX
    return coord_bounds_px

def tile_set_rx(coords_rx: np.ndarray) -> Set[Tuple[int, int]]:
    # returns coordinates of tiles involved
    tile_coords = {(x, y) for x, y in np.floor(coords_rx / TILE_SIZE_PX).astype(int).transpose()}
    return tile_coords

def meters_per_pixel(level):
    # Resolution (meters/pixel) for given zoom level (measured at Equator)
    return METERS_PER_PIXEL_L0 / nb_tiles(level)


import tempfile
import pandas as pd
from loguru import logger
from typing import BinaryIO

from app.sellers_scraper.amazon import scrape_amazon
from app.sellers_scraper.flipkart import scrape_flipkart
from app.sellers_scraper.myntra import scrape_myntra

def write_to_excel(
    amazon_sellers: list[str],
    flipkart_sellers: list[str],
    myntra_sellers: list[str],
    filename: str,
) -> None:
    amazon_df = pd.DataFrame(
        amazon_sellers,
        columns=["Rating", "Seller Name", "Seller Address"],
    )

    flipkart_df = pd.DataFrame(
        flipkart_sellers,
        columns=["Rating", "Seller Name", "Manufacturer Address", "Packer Address"],
    )

    myntra_df = pd.DataFrame(
        myntra_sellers,
        columns=[
            "Seller Name",
            "Seller Address",
            "Importer Address",
            "Manufacturer Address",
            "Packer Address",
        ],
    )

    with pd.ExcelWriter(filename) as writer:
        if amazon_sellers:
            amazon_df.drop_duplicates(inplace=True)
            amazon_df.to_excel(writer, sheet_name="Amazon", index=None)

        if flipkart_sellers:
            flipkart_df.drop_duplicates(inplace=True)
            flipkart_df.to_excel(writer, sheet_name="Flipkart", index=None)
        if myntra_sellers:
            myntra_df.drop_duplicates(inplace=True)
            myntra_df.to_excel(writer, sheet_name="Myntra", index=None)

def main(excel_file: BinaryIO, filename: str) -> None:
    with tempfile.NamedTemporaryFile(delete=False) as temp_file:
        lines = excel_file.readlines()
        temp_file.writelines(lines)
        temp_file.seek(0)

        amazon_sellers = []
        flipkart_sellers = []
        myntra_sellers = []

        df = pd.read_excel(temp_file)
        product_urls = df["URLs"].dropna().to_list()

        for product_url in product_urls:
            if "amazon" in product_url:
                amazon_sellers += scrape_amazon(product_url)
            elif "flipkart" in product_url:
                flipkart_sellers += scrape_flipkart(product_url)
            elif "myntra" in product_url:
                myntra_sellers += scrape_myntra(product_url)
            else:
                logger.error("invalid url")

        write_to_excel(amazon_sellers, flipkart_sellers, myntra_sellers, filename)


#! usr/bin/python
# -*- coding: utf-8 -*-

#Agenda con base de datos Sqlite3
#www.pythondiario.com
#Autor: Diego Caraballo

#Modulos importados
import sqlite3
import time
import os

#Conexion con Base de Datos Sqlite3
con = sqlite3.connect("agenda.db")
cursor = con.cursor()
#Comprueba si la tabla existe, en caso de no existir la crea
cursor.execute("""CREATE TABLE IF NOT EXISTS datos (nombre TEXT, telefono TEXT, correo
    TEXT, dire TEXT,empresa TEXT,departamento TEXT,  vigente TEXT, id INTEGER PRIMARY KEY AUTOINCREMENT )""")

cursor.close()

#Declaracion de las funciones

def limpiar():

    """Limpia la pantalla"""

    if os.name == "posix":
        os.system("clear")
    elif os.name == ("ce", "nt", "dos"):
        os.system("cls")

def agregar():

    """Agrega un nuevo contacto a la Agenda"""

    print ("Agregar contacto")
    print ("----------------")
    print ("")

    con = sqlite3.connect("agenda.db")
    cursor = con.cursor()

    nombre = input("Nombre: ")
    telefono = input("Telefono: ")
    correo  = input("Correo: ")
    dire    = input("Direccion: ")
    empresa    = input("Empresa: ")
    departamento    = input("departamento: ")
    vigente = input("vigente (1) No (0): ")

    cursor.execute("insert into datos (nombre, telefono, correo, dire, empresa, departamento, vigente) \
        values ('%s','%s','%s','%s','%s','%s','%s')"%(nombre,telefono,correo,dire,empresa,departamento,vigente))

    con.commit()

    print ("Los datos fueron agregados correctamente")

    cursor.close()
    time.sleep(2)
    main()

def ver():

    """Devuelve todos los contactos de la agenda"""

    print ("Lista de contactos")
    print ("------------------")
    print ("")

    con = sqlite3.connect("agenda.db")
    cursor = con.cursor()

    cursor.execute("SELECT * FROM datos")
    resultado = cursor.fetchall()

    for i in resultado:
        #print ("%s %s %s %s %s" % (i[0],i[1],i[2],i[3],i[5]))
        print (i[0].ljust(80),i[1].ljust(30),i[2].ljust(40),i[3],i[5],sep="|")

    cursor.close()

    print ("")
    input("Presione una tecla para continuar...")

    main()

def buscar():

    """Busca un contacto en la agenda y lo lista"""

    print ("Buscar contacto")
    print ("---------------")
    print ("")

    con = sqlite3.connect("agenda.db")
    cursor = con.cursor()

    buscar = input("Nombre a buscar: ")
    consulta="SELECT * FROM datos WHERE nombre LIKE '%"+buscar+"%'"
    cursor.execute (consulta)

    x = cursor.fetchall()

    print ("")

    for i in x:
        print (i[0].ljust(80),i[1].ljust(30),i[2].ljust(40),i[3],i[5],sep="|")

        print ("")

    cursor.close()

    print ("")
    input("Presione una tecla para continuar...")

    main()

def eliminar():

    """Elimina un contacto de la Agenda"""

    print ("Eliminar contacto")
    print ("-----------------")
    print ("")

    con = sqlite3.connect("agenda.db")
    cursor = con.cursor()

    eliminar = input ("Nombre de contacto a eliminar: ")

    cursor.execute("DELETE FROM datos WHERE nombre='%s'"%(eliminar))

    con.commit()

    cursor.close()

    print ("Contacto eliminao correctamente...")
    input()
    main()

def main():

    """Funcion principal de la Agenda"""

    limpiar()

    print ("-----------------------------------------")
    print ("Esta es la agenda de www.MarstonSoftware.com")
    print ("-----------------------------------------")
    print ("                              Version 1.0")
    print ("""
    [1] Ingresar Contacto
    [2] Listar Contactos
    [3] Buscar Contacto
    [4] Eliminar Contacto
    [0] Salir
    """)

    opcion = input("Ingresa una opción -> ")

    if opcion != "1" and opcion != "2" and opcion != "3" and opcion != "4" and opcion != "5" and opcion != "0":
        print ("Opcion incorrecta")
        input()
        main()
    elif opcion == "1":
        limpiar()
        agregar()
    elif opcion == "2":
        limpiar()
        ver()
    elif opcion == "3":
        limpiar()
        buscar()
    elif opcion == "4":
        limpiar()
        eliminar()
    elif  opcion == "0":
        print ("")
        print ("Bye...")
        print ("No te olvides de visitar www.MarstonSoftware.com :")
        print ("")
        print ("")
        time.sleep(3)
        exit()

main()


from pathlib import Path

import pytest
import numpy as np

from cgmml.common.depthmap_toolkit.depthmap import Depthmap
from cgmml.common.depthmap_toolkit.visualisation import blur_face, render_rgb, CHILD_HEAD_HEIGHT_IN_METERS

TOOLKIT_DIR = Path(__file__).parents[0].absolute()
OFFSET_X_Y = (0.37, 0.53)
DEPTHMAP_DIR = TOOLKIT_DIR / 'huawei_p40pro'
DEPTHMAP_FPATH = DEPTHMAP_DIR / 'depth' / 'depth_dog_1622182020448_100_282.depth'
RGB_FPATH = DEPTHMAP_DIR / 'rgb' / 'rgb_dog_1622182020448_100_282.jpg'
CALIBRATION_FPATH = str(TOOLKIT_DIR / 'huawei_p40pro' / 'camera_calibration.txt')

def test_blur_face():
    dmap = Depthmap.create_from_zip_absolute(DEPTHMAP_FPATH, RGB_FPATH, CALIBRATION_FPATH)

    # Find top of the object
    floor = dmap.get_floor_level()
    assert floor == pytest.approx(-0.9706086, 0.001)
    mask = dmap.segment_child(floor)
    highest_point = dmap.get_highest_point(mask)  # 3D

    # Render the color data
    output_unblurred = render_rgb(dmap)

    # Blur
    output_blurred = blur_face(output_unblurred, highest_point, dmap, CHILD_HEAD_HEIGHT_IN_METERS)

    # Assert some pixels in whole image change (image not same)
    all_count = dmap.width * dmap.height
    count = np.count_nonzero(output_unblurred - output_blurred) / 3
    ratio_blurred = count / all_count
    assert 0.01 < ratio_blurred < 0.9

    # Assert that blurred around object
    object_x = int(dmap.width * OFFSET_X_Y[0])
    object_y = int(dmap.height * (1.0 - OFFSET_X_Y[1]))
    slice_x = slice(object_x - 2, object_x + 2)
    slice_y = slice(object_y - 2, object_y + 2)
    assert (output_unblurred[slice_x, slice_y] != output_blurred[slice_x, slice_y]).any()

    # Assert that NOT blurred around corner
    corner_x = 0
    corner_y = 0
    slice_x = slice(corner_x, corner_x + 4)
    slice_y = slice(corner_y, corner_y + 4)
    np.testing.assert_array_equal(output_unblurred[slice_x, slice_y], output_blurred[slice_x, slice_y])

if __name__ == "__main__":
    test_blur_face()


from collections import deque

dx = [0, 0, 1, -1]
dy = [1, -1, 0, 0]

n = int(input())
a = [list(map(int, input())) for _ in range(n)]
result = [[0]*n for _ in range(n)]

q = deque()
check = [[False] * n for _ in range(n)]

def bfs(sx, sy, number):
  check[sx][sy] = True
  result[sx][sy] = number
  q.append((sx, sy))

  while q:
    x, y = q.popleft()

    for k in range(4):
      nx, ny = x+dx[k], y+dy[k]

      if 0 <= nx < n and 0 <= ny < n:
        if not check[nx][ny] and a[nx][ny] == 1:
          result[nx][ny] = number
          check[nx][ny] = True
          q.append((nx, ny))

number = 0
for i in range(n):
  for j in range(n):
    if a[i][j] == 1 and not check[i][j]:
      number += 1
      bfs(i, j, number)

print(number)

number_count = []

for i in range(1, number+1):
  count = 0
  for j in range(n):
    for k in range(n):
      if result[j][k] == i:
        count += 1

  number_count.append(count)
number_count.sort()
for el in number_count:
  print(el)


import os
import wget
import numpy as np

# Function for downloading data files from Zenodo
def download_zenodo():

   if os.path.exists("MatlabData"):
       pass
   else:
       print("Downloading files from Zenodo ... ")
       os.mkdir("MatlabData")
       # Powder Phantom Data
       wget.download("https://zenodo.org/record/5825464/files/Powder_phantom_180s_180Proj_sinogram.mat", out="MatlabData")
       wget.download("https://zenodo.org/record/5825464/files/Powder_phantom_30s_30Proj_sinogram.mat", out="MatlabData")
       wget.download("https://zenodo.org/record/5825464/files/Energy_axis.mat", out="MatlabData")
       print("\nFinished.")

def rmse_sing_chan(calculated,truth,errors):
    # Calculates RMSE for every channel of a single voxel
    for i in range(0,len(calculated)):
        errors[i] = np.sqrt(np.mean((calculated[i] - truth[i]) ** 2))
    print('Calculation Complete!')
    return errors

def cnr_spatial(mean_signal,std_signal,mean_bg,std_bg):
    # Need an ROI to do this
    ratio = (np.abs(mean_signal-mean_bg))/(std_signal+std_bg)
    print('Calculation Complete!')
    return ratio


from collections import Counter
items = ['YES', 'NO', 'NO', 'YES', 'EMPTY', 'YES', 'NO']
count = Counter()
for i in items:
    count[i]+=1
print(count)

# Another Sol
result = dict(zip(list(items),[list(items).count(i) for i in list(items)]))
print(f'Counter({result})')

200+ Exercises  Programming in Python/22- built-in modules/Ex 9.py


__author__ = 'mnowotka'

from tastypie import http
from tastypie import fields
from tastypie.resources import ALL
from chembl_webservices.core.resource import ChemblModelResource
from chembl_webservices.core.meta import ChemblResourceMeta
from chembl_webservices.core.serialization import ChEMBLApiSerializer
from chembl_webservices.core.utils import NUMBER_FILTERS, CHAR_FILTERS, FLAG_FILTERS
from django.core.exceptions import ObjectDoesNotExist
from django.db.models import Prefetch

try:
    from chembl_compatibility.models import TargetRelations
except ImportError:
    from chembl_core_model.models import TargetRelations
try:
    from chembl_compatibility.models import ChemblIdLookup
except ImportError:
    from chembl_core_model.models import ChemblIdLookup
try:
    from chembl_compatibility.models import TargetDictionary
except ImportError:
    from chembl_core_model.models import TargetDictionary

from chembl_webservices.core.fields import monkeypatch_tastypie_field
monkeypatch_tastypie_field()

# ----------------------------------------------------------------------------------------------------------------------

class TargetRelationsResource(ChemblModelResource):

    target_chembl_id = fields.CharField('target__chembl_id')
    related_target_chembl_id = fields.CharField('related_target__chembl_id')

    class Meta(ChemblResourceMeta):
        queryset = TargetRelations.objects.all()
        excludes = ['targrel_id']
        resource_name = 'target_relation'
        collection_name = 'target_relations'
        serializer = ChEMBLApiSerializer(resource_name, {collection_name: resource_name})
        detail_uri_name = 'target__chembl_id'
        prefetch_related = [
            Prefetch('target', queryset=TargetDictionary.objects.only('chembl')),
            Prefetch('related_target', queryset=TargetDictionary.objects.only('chembl')),
        ]

        fields = (
            'relationship',
        )

        filtering = {
            'target_chembl_id': NUMBER_FILTERS,
            'related_target_chembl_id': CHAR_FILTERS,
            'relationship': CHAR_FILTERS,
        }
        ordering = [field for field in filtering.keys() if not ('comment' in field or
                                                                'description' in field or 'canonical_smiles' in field)]

# ----------------------------------------------------------------------------------------------------------------------

    def get_detail_impl(self, request, basic_bundle, **kwargs):
        """
        Returns a single serialized resource.

        Calls ``cached_obj_get/obj_get`` to provide the data, then handles that result
        set and serializes it.

        Should return a HttpResponse (200 OK).
        """

        try:
            obj, in_cache = self.cached_obj_get_list(bundle=basic_bundle, **self.remove_api_resource_names(kwargs))
            objects = obj.get('target_relations', [])
        except ObjectDoesNotExist:
            return http.HttpNotFound()

        if len(objects) == 1:
            object = objects[0]
            bundle = self.build_bundle(obj=object, request=request)
            bundle = self.full_dehydrate(bundle, **kwargs)
            bundle = self.alter_detail_data_to_serialize(request, bundle)
            return self.create_response(request, bundle)
        else:
            bundles = []

            for ob in obj[self._meta.collection_name]:
                bundle = self.build_bundle(obj=ob, request=request)
                bundles.append(self.full_dehydrate(bundle, for_list=True, **kwargs))

            obj[self._meta.collection_name] = bundles
            obj = self.alter_list_data_to_serialize(request, obj)

            return obj, in_cache

# ----------------------------------------------------------------------------------------------------------------------


import numpy as np
import cv2

cat_cascade = cv2.CascadeClassifier('cascade.xml')

img = cv2.imread('Cat1.jpeg')
cats = cat_cascade.detectMultiScale(gray,1.01,7)
for(x,y,w,h) in cats:
    img = cv2.rectangle(img,(x,y),(x+w,y+h),(255,0,0),2)

cv2.imshow('cats',img)
cv2.waitKey(0)
cv2.destroyAllWindows()


"""empty message

Revision ID: 01b0cbb9a1a7
Revises: 37e499a6b25
Create Date: 2016-04-14 10:29:24.710604

"""

# revision identifiers, used by Alembic.
revision = '01b0cbb9a1a7'
down_revision = '37e499a6b25'

from alembic import op
import sqlalchemy as sa

def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.add_column('time_place', sa.Column('latitude', sa.Float(), nullable=True))
    op.add_column('time_place', sa.Column('longitude', sa.Float(), nullable=True))
    ### end Alembic commands ###

def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('time_place', 'longitude')
    op.drop_column('time_place', 'latitude')
    ### end Alembic commands ###


#!/usr/bin/env python
# encoding: utf-8
#@author: jack
#@contact: 935650354@qq.com
#@site: https://www.cnblogs.com/jackzz
#@software: PyCharm
#@time: 2019/5/30 19:11

from django.http import HttpResponseForbidden

def simple_middleware(get_response): #参数固定
    print("一次性设置和初始化1")

    def middleware(request):
        user_agent = request.META['HTTP_USER_AGENT']
        if not 'chrome' in user_agent.lower():
            return HttpResponseForbidden()
        print('处理请求之前执行的代码2')

        response = get_response(request)

        print("处理请求、响应之后的代码5")
        return response
    return middleware

class SimpleMiddleWare:
    def __init__(self,get_response):
        self.get_response = get_response
        print("一次性设置和初始化0")

    def __call__(self, request):
        print('处理请求之前执行的代码3')

        response = self.get_response(request)

        print("处理请求、响应之后的代码4")
        return response


import signal
import functools
import logging

logger = logging.getLogger('awx.main.tasks.signals')

__all__ = ['with_signal_handling', 'signal_callback']

class SignalExit(Exception):
    pass

class SignalState:
    def reset(self):
        self.sigterm_flag = False
        self.sigint_flag = False

        self.is_active = False  # for nested context managers
        self.original_sigterm = None
        self.original_sigint = None
        self.raise_exception = False

    def __init__(self):
        self.reset()

    def raise_if_needed(self):
        if self.raise_exception:
            self.raise_exception = False  # so it is not raised a second time in error handling
            raise SignalExit()

    def set_sigterm_flag(self, *args):
        self.sigterm_flag = True
        self.raise_if_needed()

    def set_sigint_flag(self, *args):
        self.sigint_flag = True
        self.raise_if_needed()

    def connect_signals(self):
        self.original_sigterm = signal.getsignal(signal.SIGTERM)
        self.original_sigint = signal.getsignal(signal.SIGINT)
        signal.signal(signal.SIGTERM, self.set_sigterm_flag)
        signal.signal(signal.SIGINT, self.set_sigint_flag)
        self.is_active = True

    def restore_signals(self):
        signal.signal(signal.SIGTERM, self.original_sigterm)
        signal.signal(signal.SIGINT, self.original_sigint)
        # if we got a signal while context manager was active, call parent methods.
        if self.sigterm_flag:
            if callable(self.original_sigterm):
                self.original_sigterm()
        if self.sigint_flag:
            if callable(self.original_sigint):
                self.original_sigint()
        self.reset()

signal_state = SignalState()

def signal_callback():
    return bool(signal_state.sigterm_flag or signal_state.sigint_flag)

def with_signal_handling(f):
    """
    Change signal handling to make signal_callback return True in event of SIGTERM or SIGINT.
    """

    @functools.wraps(f)
    def _wrapped(*args, **kwargs):
        try:
            this_is_outermost_caller = False
            if not signal_state.is_active:
                signal_state.connect_signals()
                this_is_outermost_caller = True
            return f(*args, **kwargs)
        finally:
            if this_is_outermost_caller:
                signal_state.restore_signals()

    return _wrapped


from django.shortcuts import render, get_object_or_404
from django.http import HttpResponse, HttpResponseRedirect, Http404
from urllib.parse import quote_plus

from django.urls import reverse, reverse_lazy

from django.contrib.auth.decorators import login_required

from django.contrib import messages

# local
from .forms import ArticleForm
from .models import Article, ArticleMedia

# Create your views here.

@login_required
def create(request):
    form = ArticleForm(request.POST or None, request.FILES or None)
    files = request.FILES.getlist('images')
    if form.is_valid():
        # import pdb; pdb.set_trace()
        # instance = form.save(commit=False)
        title = form.cleaned_data.get("title")
        content = form.cleaned_data.get("content")
        publish = form.cleaned_data.get("publish")

        image = files[0]

        instance = Article.objects.create(user=request.user, title=title, image = image, content=content, publish=publish)
        instance.save()

        for f in files:
            instance_fichier = ArticleMedia.objects.create(photos=f, article=instance)
            instance_fichier.save()
        messages.success(request, "L'article a bien été créé")

        return HttpResponseRedirect(instance.get_absolute_url())

    context = {
        "form": form,
    }

    return render(request, "evenements/create.html", context)

def articles(request):
    instances = Article.objects.all()

    context = {
        "instances": instances,
    }

    return render(request, "evenements/articles.html", context)

def detail(request, id=None, slug=None):
    instance = get_object_or_404(Article, id=id, slug=slug)

    share_string = quote_plus(instance.content)
    photos = ArticleMedia.objects.filter(article_id=instance.id)

    context = {
        "instance": instance,
        "photos": photos,
        "share_string": share_string,
    }

    return render(request, "evenements/detail.html", context)

@login_required
def update(request, id=None, slug=None):
    instance = get_object_or_404(Article, id=id, slug=slug)
    if instance.user != request.user:
        response = HttpResponse("Vous ne pouvez pas modifier cette publication car vous n'en êtes pas le propriétaire.")
        response.status_code = 403
        return response
    form = EvenementForm(request.POST or None, request.FILES or None, instance=instance)
    files = request.FILES.getlist("image")
    if form.is_valid():
        instance = form.save(commit=False)
        instance.save()
        for f in files:
            instance_fichier = FichierModel(photos=f, evenement=instance)
            instance_fichier.save()
        messages.success(request, "L'évenement a bien été modifié !")
        return HttpResponseRedirect(instance.get_absolute_url())

    context = {
        # "ville": instance.ville,
        "instance": instance,
        "form": form,
    }

    return render(request, "evenements/create.html", context)

@login_required
def delete(request, id, slug):
    try:
        instance = Article.objects.get(id=id, slug=slug)
    except:
        raise Http404

    if instance.user != request.user:
        response = HttpResponse("Vous ne pouvez pas supprimer l'évenement car vous n'en êtes pas le propriétaire.")
        response.status_code = 403
        return response

    if request.method == "POST":
        instance.delete()
        messages.success(request, "This has been deleted.")
        return HttpResponseRedirect(reverse_lazy("home"))

    context = {
        "instance": instance,
    }

    return render(request, "evenements/confirme_delete.html", context)


from pyrogram import Client
from datetime import datetime
import pandas as pd
import time
import os
import dotenv

dotenv.load_dotenv()

api_id = os.getenv('API_ID')
api_hash = os.getenv('API_HASH')
channel_id = 'Spotted_DMI'

# Create a Pyrogram client
app = Client('session_name', api_id=api_id, api_hash=api_hash)
started = datetime.today()

print(" ######## app started successfully ########### ")
app.start()
i = 0
data_list = []
chat_history = app.get_chat_history(channel_id, limit=100,offset_id=16108)
for message in chat_history:
    i+=1

    # A) Possibility to get the posts with images AND video and getting the text under written.
    if message.text is None:
        print("\n##### Spot numero: " + str(i) + " ID : " + str(message.id))

        if message.photo:
            print(" ------------- Image of the Spot ------------------- ")
            # A) Download the images/video on the "downloads" directory.
            image = app.download_media(message.photo.file_id,f'image{message.id}.jpg')
            print(image)
        if message.video:
            print(" ------------- Video of the Spot ------------------- ")
            video = app.download_media(message.video.file_id,f'video{message.id}.mp4')
            print(video)
            if message.caption:
                print(" ------------- Text of the Spot ------------------- ")
                text_below = message.caption
                print(message.caption)

    if message.text is not None:
        print("\n##### Spot numero: " + str(i) + " ID : " + str(message.id))
        print(message.text)
        comments = []
        print("\n## Commenti del post con ID :" + str(message.id))
        for reply in app.get_discussion_replies(channel_id, message.id):

            # Q) Not sure if is saving the corret file, not visible from my laptop.

            # if reply.text is None:
            #     if reply.animation:
            #         print(" ------------- GIF of the comment ------------------- ")
            #         gif = app.download_media(reply.animation.file_id,f'animation{reply.id}.gif')
            #         print(gif)
            #     if reply.sticker:
            #         print(" ------------- Sticker of the comment ------------------- ")
            #         sticker = app.download_media(reply.sticker.file_id,f'sticker{reply.id}.jpeg')
            #         print(sticker)

                # A) Added the same concept as the post.
                if reply.photo:
                    print(" ------------- Image of the comment ------------------- ")
                    # A) Download the images/video on the "downloads" directory.
                    image = app.download_media(reply.photo.file_id,f'image{reply.id}.jpg')
                    print(image)
                if reply.video:
                    print(" ------------- Video of the comment ------------------- ")
                    video = app.download_media(reply.video.file_id,f'video{reply.id}.mp4')
                    print(video)

                    if reply.caption:
                        print(" ------------- Text of the comment ------------------- ")
                        text_below = reply.caption
                        print("- "+ message.caption)

        # A) If the message doesn't contain text it doesn't enter in the if (None = GIF,STICKER,PHOTO,VIDEO,VOICE MESSAGE(?) ).
        if reply.text is not None:
            print("- "+reply.text)
            comments.append(reply.text)
            time.sleep(2)

        data_list.append({
            "Message ID": message.id,
            "Spot": message.text,
            "Comments": comments
        })

app.stop()

df = pd.DataFrame(data_list)
df.to_csv("data.csv", index=False)


'''
python multithreading
create sub-thread

create
start()
'''

import logging
import threading
import time

def thread_func(num):
    logging.info('sub-thread is started.')

    time.sleep(3)
    for i in range(num):
        logging.info('in sub thread - %d', i)

    logging.info('sub-thread is end.')

if __name__=='__main__':
    format = '%(asctime)s: %(message)s'
    logging.basicConfig(format=format, level=logging.INFO, datefmt="%H:%M:%S")
    logging.info("In the Main thread")

    new_thread = threading.Thread(target=thread_func, args=(100,))
    new_thread.start()

    logging.info('main thread end.')


class FreqStack:
    # O(1), O(N)
    def __init__(self):
        from collections import defaultdict
        self.c = Counter()
        self.stack = defaultdict(list)
        self.maxfreq = 0

    def push(self, x: int) -> None:
        self.c[x] +=1
        self.maxfreq = max(self.maxfreq, self.c[x])
        self.stack[self.c[x]].append(x)

    def pop(self) -> int:
        cand = self.stack[self.maxfreq].pop()
        self.c[cand] -= 1
        if not self.stack[self.maxfreq]:
            self.maxfreq -=1
        return cand

# Your FreqStack object will be instantiated and called as such:
# obj = FreqStack()
# obj.push(x)
# param_2 = obj.pop()


import os
import cv2 as cv
import shutil
import configparser
import sys

flag = False
rect = [0, 0, 0, 0]
rects=[]
img = []
img1 = []
txt = []
w = 23
h = 32
hf = False
wf = False

def Draw2(events, x, y, flags, param):
    global flag, img, img1, txt, w, h, hf, wf
    if events == cv.EVENT_LBUTTONDOWN:
        txt.write(
            str(x) + ' ' + str(y) + ' ' + str(x + w) + ' ' + str(y + h) + '\n')
        cv.rectangle(img1, (x, y), (x + w, y + h), (80, 80, 80), 1)

    if events == cv.EVENT_MOUSEMOVE:
        img3 = img1.copy()
        img = img3.copy()
        img2 = img1.copy()
        cv.rectangle(img2, (x, y), (x + w, y + h), (0, 0, 0), 1)
        img = img2.copy()

    if events == cv.EVENT_MOUSEWHEEL:
        pass

def Draw(events, x, y, flags, param):
    global flag, rect, rects, img, img1, txt
    if events == cv.EVENT_LBUTTONDOWN:
        rect[0] = x
        rect[1] = y
        flag = True
    elif events == cv.EVENT_LBUTTONUP:
        flag = False
        rects.append(rect.copy())
        #print(rects)
        cv.rectangle(img1, (rect[0], rect[1]), (rect[2], rect[3]),
                     (70, 70, 70), 1)
    if events == cv.EVENT_MOUSEMOVE:
        img3 = img1.copy()
        cv.line(img3, (0, y), (655, y), (30, 30, 30), 1)
        cv.line(img3, (x, 0), (x, 489), (30, 30, 30), 1)
        img = img3.copy()
        if flag:
            rect[2] = x
            rect[3] = y
            img2 = img1.copy()
            cv.rectangle(img2, (rect[0], rect[1]), (rect[2], rect[3]),
                         (0, 0, 0), 1)
            img = img2.copy()

def MakeConfig():
    cf=configparser.ConfigParser()
    cf.add_section('dir')
    cf.set('dir','images_dir','images')
    cf.set('dir','outputs_dir','outputs')
    cf.set('dir','tmp_dir','tmp')

    cf.add_section('label')
    cf.set('label','prefix','20181215')
    cf.set('label','postfix','T3a')
    cf.set('label','length','15')

    cf.add_section('flag')
    cf.set('flag','move2tmp','1')
    cf.set('flag','update_fix','1')

    cf.add_section('image')
    cf.set('image','type','bmp')

    cf.write(open('annotation.config','w'))

if not os.path.exists('annotation.config'):
    MakeConfig()

cf=configparser.ConfigParser()
cf.read('annotation.config')

imagesDir=cf.get('dir','images_dir')
outputsDir=cf.get('dir','outputs_dir')
tmpDir=cf.get('dir','tmp_dir')

imgtype=cf.get('image','type')

labellen=cf.getint('label','length')

isMoveToTmp=cf.getint('flag','move2tmp')
isUpdateFix=cf.getint('flag','update_fix')

if not os.path.exists(imagesDir):
    imagesDir=input('Please enter the images directory: ')
    cf.set('dir','imagesDir',images_dir)
    cf.write(open('annotation.config','w'))
if not os.path.exists(outputsDir):
    os.mkdir('outputs')
if not os.path.exists(tmpDir):
    os.mkdir('tmp')

cv.namedWindow('annotations', cv.WINDOW_NORMAL)
cv.setMouseCallback('annotations', Draw)
imgs = os.listdir(imagesDir)
cnt = len(imgs)
for imgn in imgs:
    print(imagesDir+'/'+imgn, cnt)
    cnt -= 1

    cf.read('annotation.config')
    prefix=cf.get('label','prefix')
    postfix=cf.get('label','postfix')
    print(prefix)
    lenpre=len(prefix)
    lenpost=len(postfix)

    txt = open(outputsDir+'/' + imgn.split('.')[0] + '.txt', 'w')
    img = cv.imread(imagesDir+'/' + imgn)
    img1 = img.copy()
    while (1):
        keyin = -1
        cv.imshow('annotations', img)
        keyin = cv.waitKey(100)
        #if keyin & 0xFF >0 and keyin & 0xFF<255:
         #   print(keyin)
        if keyin & 0xFF == 27:  #esc
            sys.exit()
        if keyin & 0xFF == 13 and len(prefix) == len(rects):
            #print(rects)
            for i in range(len(rects)):
                rect_=rects[i]
                txt.write((prefix+postfix)[i]+' '+str(rect_[0]) + ' ' + str(rect_[1]) + ' ' + str(rect_[2]) + ' ' +str(rect_[3]) + '\n')
            txt.close()
            if isMoveToTmp != 0:
                shutil.move(imagesDir+'/' + imgn, tmpDir+'/' + imgn)
            print('ok: '+prefix)
            rects=[]
            break
        if keyin & 0xFF == 13 and len(prefix) != len(rects):  # Enter
            txt.close()
            count = 0
            break
        if keyin & 0xFF >= 48 and keyin & 0xFF <= 120:
            prefix += chr(keyin)
            if len(prefix)==lenpre and isUpdateFix!=0:
                cf.set('label','prefix',prefix)
                cf.write(open('annotation.config','w'))
            if len(prefix)==labellen and isUpdateFix!=0:
                postfix=prefix[labellen-lenpost:]
                cf.set('label','postfix',postfix)
                cf.write(open('annotation.config','w'))
            print(prefix+'\n')
            cv.putText(img, prefix, (50, 50), cv.FONT_HERSHEY_COMPLEX, 1,
                       (0, 0, 0), 2)
        #if len(prefix+postfix) == labellen:
         #   prefix += postfix
          #  cv.putText(img, prefix, (50, 50), cv.FONT_HERSHEY_COMPLEX, 1,
           #            (0, 0, 0), 2)
        if keyin & 0xFF == 8:
            prefix = prefix[:-1]
            img5=img1.copy()
            cv.putText(img5, postfix + prefix, (80, 80), cv.FONT_HERSHEY_COMPLEX, 1,
                       (0, 0, 0), 2)
            img=img5
            print(prefix+'\n')
cv.destroyAllWindows()


import os
os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'
import sys
import argparse
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from visualizers import BCVisualizer
from data_io import read_features
from cnn import myMLP

# データフォルダ
DATA_DIR = './dataset/Example/'

# ダミーデータを対象とするか否か
USING_DMY = False

# エントリポイント
if __name__ == '__main__':

    # コマンドライン引数のパース
    parser = argparse.ArgumentParser(description='Example of Perceptron')
    parser.add_argument('--gpu', '-g', default=-1, type=int, help='GPU ID (negative value indicates CPU)')
    parser.add_argument('--epochs', '-e', default=50, type=int, help='number of epochs to learn')
    parser.add_argument('--batchsize', '-b', default=100, type=int, help='learning minibatch size')
    parser.add_argument('--v_interval', '-v', default=5, type=int, help='visualization interval')
    args = parser.parse_args()

    # デバイスの設定
    dev_str = 'cuda:{0}'.format(args.gpu) if torch.cuda.is_available() and args.gpu >= 0 else 'cpu'
    dev = torch.device(dev_str)

    # オプション情報の設定・表示
    epochs = max(1, args.epochs) # 総エポック数（繰り返し回数）
    batchsize = max(1, args.batchsize) # バッチサイズ
    visualization_interval = max(0, args.v_interval) # 何エポックごとに可視化結果を表示するか
    print('device: {0}'.format(dev_str), file=sys.stderr)
    print('epochs: {0}'.format(epochs), file=sys.stderr)
    print('batchsize: {0}'.format(batchsize), file=sys.stderr)
    print('visualization interval: {0}'.format(visualization_interval), file=sys.stderr)
    print('', file=sys.stderr)

    # データの読み込み
    if USING_DMY == False:
        # 男女別身長／体重データを対象とする場合
        labels, features, labeldict, labelnames = read_features(DATA_DIR + 'whg_train.csv') # 学習データ読み込み
        labels_ev, features_ev, labeldict, dmy = read_features(DATA_DIR + 'whg_test.csv', dic=labeldict) # 評価用データ読み込み
    else:
        # ダミーデータを対象とする場合
        labels, features, labeldict, labelnames = read_features(DATA_DIR + 'dmy_train.csv') # 学習データ読み込み
        labels_ev, features_ev, labeldict, dmy = read_features(DATA_DIR + 'dmy_test.csv', dic=labeldict) # 評価用データ読み込み
    labels = np.asarray(labels, dtype=np.int32)
    labels_ev = np.asarray(labels_ev, dtype=np.int32)
    features = np.asarray(features, dtype=np.float32)
    features_ev = np.asarray(features_ev, dtype=np.float32)
    n_samples = features.shape[0] # 学習データの総数
    n_samples_ev = features_ev.shape[0] # 評価用データの総数
    n_dims = features.shape[1] # データの次元数
    n_classes = len(labeldict) # クラス数

    # 二クラス識別器の作成
    model = myMLP(n_dims)
    model = model.to(dev)

    # 可視化の準備
    NUM = 100 # 可視化結果に重畳する実データの数
    a = np.min(features_ev, axis=0)
    b = np.max(features_ev, axis=0)
    perm = np.random.permutation(n_samples_ev)
    hrange = ((11 * a[0] - b[0]) / 10, (11 * b[0] - a[0]) / 10) # 可視化結果における横軸の範囲
    vrange = ((11 * a[1] - b[1]) / 10, (11 * b[1] - a[1]) / 10) # 可視化結果における縦軸の範囲
    data = [labels_ev[perm[ : NUM]], features_ev[perm[ : NUM]]] # 可視化結果に重畳する実データ（評価用データからランダムに NUM 個を選択）
    if USING_DMY == False:
        visualizer_hlabel = 'height (cm)' # 可視化結果における横軸のラベル
        visualizer_vlabel = 'weight (kg)' # 可視化結果における縦軸のラベル
    else:
        visualizer_hlabel = 'data 1'
        visualizer_vlabel = 'data 2'
    visualizer = BCVisualizer(hrange=hrange, vrange=vrange, hlabel=visualizer_hlabel, vlabel=visualizer_vlabel, clabels=labelnames)

    # 初期状態の可視化
    if visualization_interval > 0:
        visualizer.show(model, device=dev, samples=data, title='Initial State') # グラフを表示

    # 損失関数の定義
    loss_func = nn.CrossEntropyLoss() # softmax cross entropy

    # オプティマイザーの用意
    optimizer = optim.Adam(model.parameters())

    # 学習処理ループ
    for e in range(epochs):

        # 現在のエポック番号を表示
        print('Epoch {0}'.format(e + 1), file=sys.stderr)

        # 損失関数の値が小さくなるように識別器のパラメータを更新
        model.train()
        perm = np.random.permutation(n_samples)
        sum_loss = 0
        for i in range(0, n_samples, batchsize):
            model.zero_grad()
            x = torch.tensor(features[perm[i : i + batchsize]], device=dev)
            t = torch.tensor(labels[perm[i : i + batchsize]], device=dev, dtype=torch.long)
            loss = loss_func(model(x), t)
            loss.backward()
            optimizer.step()
            sum_loss += float(loss) * len(x)
            del loss
            del t
            del x

        # 損失関数の現在値を表示
        print('  train loss = {0:.4f}'.format(sum_loss / n_samples), file=sys.stderr)

        # 評価用データに対する識別精度を計算・表示
        model.eval()
        n_failed = 0
        for i in range(0, n_samples_ev, batchsize):
            x = torch.tensor(features_ev[i : i + batchsize], device=dev)
            t = torch.tensor(labels_ev[i : i + batchsize], device=dev, dtype=torch.long)
            y = model(x)
            y = y.to('cpu').detach().numpy().copy()
            t = t.to('cpu').detach().numpy().copy()
            n_failed += np.count_nonzero(np.argmax(y, axis=1) - t)
            del y
            del x
            del t
        acc = (n_samples_ev - n_failed) / n_samples_ev
        print('  accuracy = {0:.2f}%'.format(100 * acc), file=sys.stderr)

        # 現状態の可視化
        if visualization_interval > 0 and (e + 1) % visualization_interval == 0:
            visualizer.show(model, device=dev, samples=data, title='Epoch {0}'.format(e + 1)) # グラフを表示

        print('', file=sys.stderr)

    print('', file=sys.stderr)


from typing import Optional, List, Dict, Tuple

import pandas as pd
from sklearn.preprocessing import KBinsDiscretizer

from monitoring.measurements import (
    TotalVarianceDistance,
    HellingerDistance,
    KullbackLeiblerDivergence,
)

class VirtualDrift:
    def __init__(
        self,
        bin_count: int,
        feature_cols: Optional[List[str]],
        prediction_col: Optional[str],
        label_col: Optional[str],
    ):
        self.feature_cols = feature_cols
        self.prediction_col = prediction_col
        self.label_col = label_col
        self.bin_count = bin_count
        self.discretizers: Dict[str, KBinsDiscretizer] = {}

    def compute_drift(
        self,
        first_dataset: pd.DataFrame,
        second_dataset: pd.DataFrame,
    ) -> Dict[str, float]:
        self.init_discritizers(first_dataset)
        self.discritize(first_dataset, second_dataset)
        return self.compute_drift_measures(first_dataset, second_dataset)

    def init_discritizers(self, base_data):
        continuous_features = base_data.select_dtypes(["float"])
        for feature in continuous_features.columns:
            if feature not in self.discretizers:
                self.discretizers[feature] = self.fit_discretizer(
                    feature, continuous_features
                )

    def fit_discretizer(
        self, feature_name: str, continuous_features: pd.DataFrame
    ) -> KBinsDiscretizer:
        discretizer = KBinsDiscretizer(
            n_bins=self.bin_count, encode="ordinal", strategy="uniform"
        )
        discretizer.fit(continuous_features.loc[:, feature_name].values.reshape(-1, 1))
        return discretizer

    def discritize(self, base_data, current_data):
        for feature, discretizer in self.discretizers.items():
            base_data[feature] = discretizer.transform(
                base_data.loc[:, feature].values.reshape(-1, 1)
            )
            current_data[feature] = discretizer.transform(
                current_data.loc[:, feature].values.reshape(-1, 1)
            )
            base_data[feature] = base_data[feature].astype("int")
            current_data[feature] = current_data[feature].astype("int")

    def compute_drift_measures(
        self,
        base_data,
        current_data,
    ) -> Dict[str, float]:
        feature_drift = self.compute_feature_drift(base_data, current_data)
        prediction_drift = self.compute_prediction_drift(base_data, current_data)
        label_drift = self.compute_label_drift(base_data, current_data)

        drift_measures = {**feature_drift, **label_drift, **prediction_drift}

        # Normalize inf values
        for key, value in drift_measures.items():
            if value == float("inf"):
                drift_measures[key] = 10

        return drift_measures

    def compute_feature_drift(
        self,
        base_data: pd.DataFrame,
        current_data: pd.DataFrame,
    ) -> Dict[str, float]:
        if self.feature_cols is None:
            return {}

        base_features = base_data[self.feature_cols]
        current_features = current_data[self.feature_cols]
        features_t, features_u = self.to_observations(base_features, current_features)

        return {
            "features_tvd": TotalVarianceDistance(features_t, features_u).compute(),
            "features_hd": HellingerDistance(features_t, features_u).compute(),
            "features_kld": KullbackLeiblerDivergence(features_t, features_u).compute(),
        }

    def compute_prediction_drift(
        self,
        base_data: pd.DataFrame,
        current_data: pd.DataFrame,
    ) -> Dict[str, float]:
        if self.prediction_col is None:
            return {}

        base_predictions = pd.DataFrame(base_data.loc[:, self.prediction_col])
        current_predictions = pd.DataFrame(current_data.loc[:, self.prediction_col])
        prediction_t, prediction_u = self.to_observations(
            base_predictions, current_predictions
        )

        return {
            "prediction_tvd": TotalVarianceDistance(
                prediction_t, prediction_u
            ).compute(),
            "prediction_hd": HellingerDistance(prediction_t, prediction_u).compute(),
            "prediction_kld": KullbackLeiblerDivergence(
                prediction_t, prediction_u
            ).compute(),
        }

    def compute_label_drift(
        self, base_data: pd.DataFrame, current_data: pd.DataFrame
    ) -> Dict[str, float]:
        if self.label_col is None:
            return {}

        base_labels = pd.DataFrame(base_data.loc[:, self.label_col])
        current_labels = pd.DataFrame(current_data.loc[:, self.label_col])
        labels_t, labels_u = self.to_observations(base_labels, current_labels)

        return {
            "prediction_tvd": TotalVarianceDistance(labels_t, labels_u).compute(),
            "prediction_hd": HellingerDistance(labels_t, labels_u).compute(),
            "prediction_kld": KullbackLeiblerDivergence(labels_t, labels_u).compute(),
        }

    @staticmethod
    def to_observations(
        t: pd.DataFrame, u: pd.DataFrame
    ) -> Tuple[pd.Series, pd.Series]:
        t = (
            t.apply(
                lambda row: f"{'_'.join([str(row[val]) for val in t.columns])}", axis=1
            )
            .value_counts()
            .sort_index()
        )
        u = (
            u.apply(
                lambda row: f"{'_'.join([str(row[val]) for val in u.columns])}", axis=1
            )
            .value_counts()
            .sort_index()
        )

        joined_uniques = pd.DataFrame([t, u]).T.fillna(0).sort_index()
        joined_uniques.columns = ["t", "u"]

        t_obs = joined_uniques.loc[:, "t"]
        u_obs = joined_uniques.loc[:, "u"]

        t_pdf = t_obs / t_obs.sum()
        u_pdf = u_obs / u_obs.sum()

        return t_pdf, u_pdf


from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import TimeSeriesSplit
from sklearn.metrics import mean_squared_error
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

class BlockingTimeSeriesSplit():
    def __init__(self, n_splits):
        self.n_splits = n_splits

    def get_n_splits(self, X, y, groups):
        return self.n_splits

    def split(self, X, y=None, groups=None):
        n_samples = len(X)
        k_fold_size = n_samples // self.n_splits
        indices = np.arange(n_samples)

        margin = 0
        for i in range(self.n_splits):
            start = i * k_fold_size
            stop = start + k_fold_size
            mid = start + int(0.9 * (stop - start))
            yield indices[start: mid], indices[mid + margin: stop]

def grid_search_param(X, y, estimator, grid, k_fold, verbose=1):
    tscv = TimeSeriesSplit(n_splits=k_fold, gap=3, test_size=4)
    RSCV = GridSearchCV(estimator=estimator,
                        param_grid=grid,
                        cv=tscv,
                        n_jobs=-1,
                        scoring='neg_root_mean_squared_error',
                        refit=True,
                        verbose=verbose,
                        pre_dispatch=8,
                        error_score=-999,
                        return_train_score=True)
    RSCV.fit(X, y)
    return RSCV.best_params_, RSCV.best_score_, RSCV.best_estimator_

def optimal_params(X, y, estimator, grid, fold_range, verbose=1):
    best_score = -100000
    for k_fold in fold_range:
        param, score, model = grid_search_param(X,
                                         y,
                                         estimator,
                                         grid,
                                         k_fold,
                                         verbose)
        # scoring: neg_root_mean_squared_error
        if score > best_score:
            best_score = score
            best_param = param
            k = k_fold
            best_model = model

    return k, best_param, best_score, model

def fit_check(y, y_pred, plot=True):
    rmse = np.sqrt(mean_squared_error(y, y_pred))
    if plot:
        plt.plot(y.index, y, label="true value")
        plt.legend()
        plt.xticks(y.index[::100])
        plt.xlabel("")
        plt.ylabel("bitcoin return")
        plt.plot(y.index, y_pred, label="prediction")
        plt.legend()
        plt.show()
    return rmse

def reg_feature_select(coef, features, level=0):
    identify = coef != 0
    model_features = features[identify]
    model_coef = coef[identify]
    model_importance = pd.DataFrame({'features': model_features,
                                     'coef': model_coef}).sort_values('coef', key=abs)

    aggregate = model_importance.coef.apply(lambda x: abs(x)).sum()
    model_importance['importance_ratio'] = model_importance.coef.apply(lambda x: abs(x)/aggregate)
    index = np.where((model_importance['importance_ratio'] >= level).to_numpy() == True)
    model_importance = model_importance.iloc[index]
    plt.figure(figsize=(20,10))
    plt.barh(model_importance.features, model_importance.importance_ratio)
    plt.xlabel("importance")
    plt.ylabel("features")
    plt.show()
    return model_importance

def pca_analysis(X, y, model):
    scaler = StandardScaler()
    y_scaled = pd.DataFrame(scaler.fit_transform(y),
                            columns=y.columns)
    pca_range = range(2, 70)
    rmses = []

    for i in pca_range:
        pca = PCA(n_components=i).fit_transform(X)
        columns = [f"pca{i}" for i in range(pca.shape[1])]
        X_pca = pd.DataFrame(pca, columns=columns)

        model = model.fit(X_pca, y_scaled)
        y_pred = pd.DataFrame(model.predict(X_pca))
        y_pred = pd.DataFrame(scaler.inverse_transform(y_pred), columns=y.columns)
        rmses.append(fit_check(y, y_pred, plot=False))
    plt.plot(rmses)
    plt.xlabel("number of PCA")
    plt.ylabel("RMSE")
    plt.show()
    return rmses

def tree_feature_select(model, features, level):
    feature_importance = pd.DataFrame(
        model.feature_importances_,
        index=features,
        columns=["importance"]).sort_values("importance", ascending=True
    )
    index = np.where((feature_importance.importance >= level).to_numpy() == True)
    df = feature_importance.iloc[index]
    plt.barh(df.index, df.importance)

    return feature_importance.iloc[index].index

def output_html(path, table):
    with open(path,'w') as f:
        f.write(table.as_html())