from sys import excepthook
from selenium.webdriver import Edge
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.keys import Keys
import time
import os
import re
import openpyxl
from openpyxl import load_workbook

#GLOBAL VAR---------------------------------------------------------------------------
SLEEPTIME = 20
SITE = "https://bianalytics.rci.rogers.com/t/BI/views/DailyPostpaidReportNewSubscriberSource/Postpaid?:embed=yes#1"
dash = "-------------------"
DATE = ""
SEGMENT_NAME = ""
PROVINCE_NAME = ""
BRAND_NAME = ""
BRAND_DICT = dict([
    (0,"Rogers"),
    (1,"Fido"),
    (2,"Cityfone")
])

SEGMENT_DICT = dict([
    (0,"Consumer"),
    (1,"Rogers_for_Business")
])

PROVINCE_DICT = dict([
    (0,"BC"),
    (1,"AB"),
    (2,"SK"),
    (3,"MB"),
    (4,"ON"),
    (5,"PQ"),
    (6,"NB"),
    (7,"NS"),
    (8,"PE"),
    (9,"NL")
])
MONTH_DICT_LEAP = dict([
    (1,"31"),
    (2,"29"),
    (3,"31"),
    (4,"30"),
    (5,"31"),
    (6,"30"),
    (7,"31"),
    (8,"31"),
    (9,"30"),
    (10,"31"),
    (11,"30"),
    (12,"31")
])
MONTH_DICT = dict([
    (1,"31"),
    (2,"28"),
    (3,"31"),
    (4,"30"),
    (5,"31"),
    (6,"30"),
    (7,"31"),
    (8,"31"),
    (9,"30"),
    (10,"31"),
    (11,"30"),
    (12,"31")
])
COUNT = 2
DOWNLOAD_DIRECTORY = ""
#ROOT = "C:\\Users\\Steven.Shan\\mypython\\tableau\\web scrape\\"
ROOT = "C:\\Users\\Steven.Shan\\mypython\\tableau\\postpaid\\"
#ROOT = "C:\\Users\\Wei.Li1\\Documents\\Postpaid\\"

#--------------------------------------------------------------------------------------------------------------------------
#initialize webdriver
options = webdriver.EdgeOptions()
options.use_chromium = True
options.add_argument("--start-maximized")
# options.add_argument("--headless")
# options.add_argument("disable-gpu")

#options.add_argument("--log-level=2")

#helpers:

#check whether the xpath exist in current webpage
def check_exists_by_xpath(xpath,web):
    try:
        web.find_element(By.XPATH, xpath)
    except NoSuchElementException:
        return False
    return True

#download and add downloaded info to the map excel sheet
def download(web):
    global PROVINCE_NAME
    global COUNT
    time.sleep(SLEEPTIME)
    el = WebDriverWait(web,300).until(EC.presence_of_element_located((By.XPATH, '//*[@id="download-ToolbarButton"]')))
    el.click()
    time.sleep(SLEEPTIME)
    el = WebDriverWait(web,300).until(EC.presence_of_element_located((By.XPATH, '//*[@id="DownloadDialog-Dialog-Body-Id"]/div/fieldset/button[3]')))
    el.click()
    time.sleep(SLEEPTIME)
    el = WebDriverWait(web,300).until(EC.presence_of_element_located((By.XPATH, '//*[@id="export-crosstab-options-dialog-Dialog-BodyWrapper-Dialog-Body-Id"]/div/div[1]/div[2]/div/div/div[2]/div')))
    el.click()
    time.sleep(SLEEPTIME)
    download = web.find_element(By.XPATH, '//*[@id="export-crosstab-options-dialog-Dialog-BodyWrapper-Dialog-Body-Id"]/div/div[3]/button')
    # download = WebDriverWait(web,300).until(EC.presence_of_element_located((By.XPATH, '//*[@id="export-crosstab-options-dialog-Dialog-BodyWrapper-Dialog-Body-Id"]/div/div[3]/button')))
    download.click()
    time.sleep(SLEEPTIME)
    month = str(re.match('^[^/]*', DATE).group(0))
    year = DATE[-4:]

    try:

        os.rename(DOWNLOAD_DIRECTORY+"\\"+"Closing SC.xlsx",DOWNLOAD_DIRECTORY+"\\"+month+"_"+year+"_"+SEGMENT_NAME+"_"+PROVINCE_NAME+"_"+BRAND_NAME+".xlsx")

        print(dash+"creating map"+dash)

        wb = load_workbook(filename= DOWNLOAD_DIRECTORY+"\\MAP.xlsx")
        sheet = wb.active
        sheet["A"+str(COUNT)] = COUNT-1
        sheet["B"+str(COUNT)] = month+"_"+year+"_"+SEGMENT_NAME+"_"+PROVINCE_NAME+"_"+BRAND_NAME
        sheet["C"+str(COUNT)] = SEGMENT_NAME
        sheet["D"+str(COUNT)] = PROVINCE_NAME
        sheet["E"+str(COUNT)] = BRAND_NAME
        wb.save(filename = DOWNLOAD_DIRECTORY+"\\MAP.xlsx")

        COUNT = COUNT + 1
    except:
        COUNT=COUNT

#start SCRAPE!----------------------------------------------------------------------------------------------

#expand segment filter
def expand_Segment_filter(web):
    el = WebDriverWait(web,30).until(EC.presence_of_element_located((By.XPATH, '//*[@id="tableau_base_widget_LegacyCategoricalQuickFilter_1"]/div/div[3]/span')))
    el.click()
    time.sleep(SLEEPTIME)
#click segment all
def click_Segment_all(web):
    el = WebDriverWait(web,30).until(EC.presence_of_element_located((By.XPATH, '//*[@id="FI_federated.0dn4toz0kzraq91gkruzo1k7pvrl,Segment_113277616459342307234_834514642350544244_(All)"]/div[2]/input')))
    el.click()
    time.sleep(SLEEPTIME)

#traverse the province section and download
#special cases:
# 1. might not exist any province, download button does not exist
# 2. only one province exist, will influence how clicking convention work
def traverse_Segment(web):
    global DATE
    global SEGMENT_NAME
    global PROVINCE_NAME
    global BRAND_NAME
    global DOWNLOAD_DIRECTORY
    time.sleep(SLEEPTIME)
    expand_Segment_filter(web)
    if check_exists_by_xpath('//*[@id="FI_federated.0dn4toz0kzraq91gkruzo1k7pvrl,Segment_113277616459342307234_834514642350544244_0"]/div[2]/input',web):
        if check_exists_by_xpath('//*[@id="FI_federated.0dn4toz0kzraq91gkruzo1k7pvrl,Segment_113277616459342307234_834514642350544244_1"]/div[2]/input' , web):
            el = WebDriverWait(web,30).until(EC.presence_of_element_located((By.XPATH, '/html/body/div[5]')))
            el.click()
            time.sleep(SLEEPTIME)
            SEGMENT_NAME = "ALL"
            print(dash+"DOWNLOADING"+DATE+"_"+SEGMENT_NAME+BRAND_NAME+PROVINCE_NAME+dash)
            download(web)
            i=0
            while i<2:
                expand_Segment_filter(web)
                if check_exists_by_xpath('//*[@id="FI_federated.0dn4toz0kzraq91gkruzo1k7pvrl,Segment_113277616459342307234_834514642350544244_'+str(i)+'"]/div[2]/input' ,web):
                    #if i==0 no need to click twice
                    if i>0:
                        click_Segment_all(web)
                    #for all click at least once to reset list to empty
                    click_Segment_all(web)
                    el = WebDriverWait(web,30).until(EC.presence_of_element_located((By.XPATH, '//*[@id="FI_federated.0dn4toz0kzraq91gkruzo1k7pvrl,Segment_113277616459342307234_834514642350544244_'+str(i)+'"]/div[2]/input')))
                    el.click()
                    time.sleep(SLEEPTIME)
                    SEGMENT_NAME = web.find_element(By.XPATH, '//*[@id="FI_federated.0dn4toz0kzraq91gkruzo1k7pvrl,Segment_113277616459342307234_834514642350544244_'+str(i)+'"]/div[2]/a').get_attribute("title")
                    time.sleep(SLEEPTIME)
                    el = WebDriverWait(web,30).until(EC.presence_of_element_located((By.XPATH, '/html/body/div[5]')))
                    el.click()
                    # web.refresh()
                    time.sleep(SLEEPTIME)
                    print(dash+"DOWNLOADING"+DATE+"_"+SEGMENT_NAME+"_"+BRAND_NAME+"_"+PROVINCE_NAME+dash)
                    download(web)
                    i+=1
                else:
                    el = WebDriverWait(web,30).until(EC.presence_of_element_located((By.XPATH, '/html/body/div[5]')))
                    el.click()
                    # web.refresh()
                    time.sleep(SLEEPTIME)
                    break
            #get back to all
            expand_Segment_filter(web)
            #select all
            click_Segment_all(web)
            el = WebDriverWait(web,50).until(EC.presence_of_element_located((By.XPATH, '/html/body/div[5]')))
            el.click()
            time.sleep(SLEEPTIME)
        else:
            print(dash+"only one sheet is available(special case)"+dash)
            SEGMENT_NAME = web.find_element(By.XPATH, '//*[@id="FI_federated.07uceah05f3pzg12vx7ye10ec8da,none:PROVINCE_NM:nk1863049714700099538_14391167526842317961_0"]/div[2]/a').get_attribute("title")
            print(dash+"DOWNLOADING"+DATE+"_"+SEGMENT_NAME+"_"+BRAND_NAME+"_"+PROVINCE_NAME+dash)
            el = WebDriverWait(web,30).until(EC.presence_of_element_located((By.XPATH, '/html/body/div[5]')))
            el.click()
            time.sleep(SLEEPTIME)
            download(web)

    else:
        print(dash+"nothing to download"+dash)
        el = WebDriverWait(web,30).until(EC.presence_of_element_located((By.XPATH, '/html/body/div[5]')))
        el.click()
        time.sleep(SLEEPTIME)

#--------------------------------------------------------------------------------------------------------------------------
#expand Province filter
def expand_Province_filter(web):
    el = WebDriverWait(web,30).until(EC.presence_of_element_located((By.XPATH, '//*[@id="tableau_base_widget_LegacyCategoricalQuickFilter_3"]/div/div[3]/span')))
    el.click()
    time.sleep(SLEEPTIME)
#click segment all
def click_Province_all(web):
    el = WebDriverWait(web,30).until(EC.presence_of_element_located((By.XPATH, '//*[@id="FI_federated.0dn4toz0kzraq91gkruzo1k7pvrl,none:PROVINCE:nk13277616459342307234_834514642350544244_(All)"]/div[2]/input')))
    el.click()
    time.sleep(SLEEPTIME)

#traverse brand section and call traverse province for each brand
#all the brands will always exist
def traverse_Province(web):
    global DATE
    global SEGMENT_NAME
    global PROVINCE_NAME
    global BRAND_NAME
    global DOWNLOAD_DIRECTORY
    time.sleep(SLEEPTIME)
    PROVINCE_NAME = "ALL"
    print(dash+"Scraping"+DATE+"_"+PROVINCE_NAME+"_"+BRAND_NAME+dash)
    traverse_Segment(web)
    j=0
    while j < 10:
        PROVINCE_NAME = PROVINCE_DICT[j]
        print(dash+"Scraping"+DATE+"_"+PROVINCE_NAME+"_"+BRAND_NAME+dash)
        expand_Province_filter(web)
        if j>0:
            click_Province_all(web)
        click_Province_all(web)
        el = WebDriverWait(web,50).until(EC.presence_of_element_located((By.XPATH, '//*[@id="FI_federated.0dn4toz0kzraq91gkruzo1k7pvrl,none:PROVINCE:nk13277616459342307234_834514642350544244_'+str(j)+'"]/div[2]/input')))
        el.click()
        time.sleep(SLEEPTIME)
        el = WebDriverWait(web,30).until(EC.presence_of_element_located((By.XPATH, '/html/body/div[5]')))
        el.click()
        time.sleep(SLEEPTIME)
        traverse_Segment(web)
        j+=1
    #get back to all
    expand_Province_filter(web)
    #select all
    click_Province_all(web)
    el = WebDriverWait(web,30).until(EC.presence_of_element_located((By.XPATH, '/html/body/div[5]')))
    el.click()
    time.sleep(SLEEPTIME)

#--------------------------------------------------------------------------------------------------------------
#type date in the box
def input_date(web):
    el = WebDriverWait(web,30).until(EC.presence_of_element_located((By.XPATH, '//*[@id="typein_[Parameters].[Report Date Parameter]"]/span[1]/input')))
    el.send_keys(Keys.CONTROL + "a")
    el.send_keys(Keys.DELETE)
    time.sleep(SLEEPTIME)
    el.send_keys(DATE)
    time.sleep(SLEEPTIME)
#expand brand filter
def expand_brand_filter(web):
    el = WebDriverWait(web,30).until(EC.presence_of_element_located((By.XPATH, '//*[@id="tableau_base_widget_LegacyCategoricalQuickFilter_5"]/div/div[3]/span')))
    el.click()
    time.sleep(SLEEPTIME)
#click brand all button
def click_brand_all(web):
    el = WebDriverWait(web,50).until(EC.presence_of_element_located((By.XPATH, '//*[@id="FI_federated.0dn4toz0kzraq91gkruzo1k7pvrl,Brand13277616459342307234_834514642350544244_(All)"]/div[2]/input')))
    el.click()
    time.sleep(SLEEPTIME)

#traverse segment section will call traverse brand for each segment
def traverse_brand():
    global DATE
    global SEGMENT_NAME
    global PROVINCE_NAME
    global BRAND_NAME
    global DOWNLOAD_DIRECTORY
    time.sleep(SLEEPTIME)
    #print(dash+"Scraping Segment all")
    web = Edge(options=options)
    web.get(SITE)
    time.sleep(15)
    input_date(web)
    print(dash+"Scraping"+DATE+dash)
    time.sleep(SLEEPTIME)
    BRAND_NAME = "ALL"
    print(dash+"Scraping"+DATE+"_"+BRAND_NAME+dash)
    traverse_Province(web)
    k=0
    while k<3:
        web.quit()
        web = Edge(options=options)
        web.get(SITE)
        time.sleep(15)
        input_date(web)
        print(dash+"Scraping"+DATE+dash)
        expand_brand_filter(web)
        click_brand_all(web)
        el = WebDriverWait(web,20).until(EC.presence_of_element_located((By.XPATH, '//*[@id="FI_federated.0dn4toz0kzraq91gkruzo1k7pvrl,Brand13277616459342307234_834514642350544244_'+str(k)+'"]/div[2]/input')))
        el.click()
        time.sleep(SLEEPTIME)
        BRAND_NAME = BRAND_DICT[k]
        el = WebDriverWait(web,30).until(EC.presence_of_element_located((By.XPATH, '/html/body/div[5]')))
        el.click()
        time.sleep(SLEEPTIME)
        print(dash+"Scraping"+DATE+"_"+SEGMENT_NAME+dash)
        traverse_Province(web)
        k+=1

    #get back to all
    expand_brand_filter(web)
    #select all
    click_brand_all(web)
    el = WebDriverWait(web,20).until(EC.presence_of_element_located((By.XPATH, '/html/body/div[5]')))
    el.click()
    time.sleep(SLEEPTIME)
    print(dash+DATE+" is successfully scrapped"+dash)

#---------------------------------------------------------------------------------------------
#user interface

print(dash+"Enter the year you want to download: "+dash)
year = str(input())
print(dash+"Enter the month you want to download or -1 to download the entire year:"+dash)
month = int(input())

if month != -1:
    if int(year)%4==0:
        DATE = str(month)+"/"+MONTH_DICT_LEAP[month]+"/"+year
    else:
        DATE = str(month)+"/"+MONTH_DICT[month]+"/"+year
    COUNT = 2
    DOWNLOAD_DIRECTORY = ROOT+year+"_"+str(month)
    os.makedirs(DOWNLOAD_DIRECTORY)
    print(dash+"folder should be created"+dash)
    print(dash+"creating map"+dash)
    wb = openpyxl.Workbook()
    wb.save(DOWNLOAD_DIRECTORY+"\\MAP.xlsx")
    wb = load_workbook(filename= DOWNLOAD_DIRECTORY+"\\MAP.xlsx")
    sheet = wb.active
    sheet["B1"] = "Selection#"
    sheet["C1"] = "Segment"
    sheet["D1"] = "Province"
    sheet["E1"] = "Brand"
    wb.save(filename = DOWNLOAD_DIRECTORY+"\\MAP.xlsx")
    options.add_experimental_option("prefs", {
        "download.default_directory": DOWNLOAD_DIRECTORY
    })
    traverse_brand()
    print(dash+"successfully_scrapped   " +str(DATE)+"   "+dash)
else:
    DOWNLOAD_DIRECTORY = ROOT+year
    os.makedirs(DOWNLOAD_DIRECTORY)
    print(dash+"folder should be created"+dash)
    print(dash+"creating map"+dash)
    wb = openpyxl.Workbook()
    wb.save(DOWNLOAD_DIRECTORY+"\\MAP.xlsx")
    wb = load_workbook(filename= DOWNLOAD_DIRECTORY+"\\MAP.xlsx")
    sheet = wb.active
    sheet["B1"] = "Selection#"
    sheet["C1"] = "Segment"
    sheet["D1"] = "Province"
    sheet["E1"] = "Brand"
    wb.save(filename = DOWNLOAD_DIRECTORY+"\\MAP.xlsx")
    options.add_experimental_option("prefs", {
        "download.default_directory": DOWNLOAD_DIRECTORY
    })
    COUNT = 2
    x = 1
    while x < 13:
        if year%4==0:
            DATE = str(month)+"/"+MONTH_DICT_LEAP[x]+"/"+year
        else:
            DATE = str(month)+"/"+MONTH_DICT[x]+"/"+year
        traverse_brand()
        print(dash+"successfully_scrapped   " +str(DATE)+"   "+dash)
        x=x+1

print(dash+"Have A Great Day!"+dash)


import json

import hydra
import pandas as pd
from hydra.core.config_store import ConfigStore

from config import Training
import src

logger = src.init_logger("predict_logger", "logs/predict.log")

cs = ConfigStore.instance()
cs.store(name="predict", node=Training)

model_name = {"DT": "DecisionTreeClassifier",
              "GNB": "GaussianNB",
              "KNN": "KNeighborsClassifier",
              "LR": "LogisticRegression",
              "RF": "RandomForestClassifier"}

@hydra.main(version_base=None, config_path="configs", config_name="train_conf")
def predicting(cfg: Training) -> None:
    x_test = pd.read_csv(cfg.data.x_test, index_col=0)
    logger.info(f"Read x_test, it's shape {x_test.shape}")
    y_test = pd.read_csv(cfg.data.y_test, index_col=0).squeeze()
    logger.info(f"Read y_test, it's shape {y_test.shape}")

    for name, full_name in model_name.items():
        if cfg.params.model_type == name:
            cfg.params.model_type = full_name

    model_file_path = cfg.save_paths.models + cfg.params.model_type + ".sav"
    logger.info(f"Load model from {model_file_path}")
    model = src.load_model(model_file_path)

    logger.info("Start predict.")
    pred = src.predict_model(model, x_test)
    logger.info("End predict.")
    metrics_file_path = cfg.save_paths.metrics + cfg.params.model_type + ".json"
    logger.info(f"Save metrics to {metrics_file_path}")
    with open(metrics_file_path, "w") as file:
        json.dump(src.evaluate_model(pred, y_test), file)

if __name__ == "__main__":
    predicting()


# Blockchain

from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes

class different_class:

    string = None
    num = 1999

    def __init__(self, mystring):
        self.string = mystring

    def __repr__(self):
        return self.string + ' ' + str(self.num)

class Block:

    data = None
    previous_hash = None
    previous_block = None

    def __init__(self, data, previous_block):
        self.data = data
        self.previous_block = previous_block
        if previous_block != None:
            self.previous_hash = previous_block.compute_hash()

    def compute_hash(self):
        digest = hashes.Hash(hashes.SHA256(), backend=default_backend())
        digest.update(bytes(str(self.data), 'utf-8'))
        digest.update(bytes(str(self.previous_hash), 'utf8'))
        return digest.finalize()

    def is_valid(self):
        if self.previous_block == None:
        	return True
        return self.previous_block.compute_hash() == self.previous_hash

if __name__ == '__main__':
    root = Block('I am root', None)
    B1 = Block('I am a child.', root)
    B2 = Block('I am B1s brother', root)
    B3 = Block(12354, B1)
    B4 = Block(different_class('Hi there!'), B3)
    B5 = Block('Top block', B4)

    for b in [B1, B2, B3, B4, B5]:
        if b.previous_block.compute_hash() == b.previous_hash:
            print ('Success! Hash is good.')
        else:
            print ('Error! Hash is no good.')

    B3.data = 12345
    if B4.previous_block.compute_hash() == B4.previous_hash:
        print ("Error! Couldn't detect tampering.")
    else:
        print ('Success! Tampering detected.')

    print (B4.data)
    B4.data.num = 1988
    print (B4.data)
    if B5.previous_block.compute_hash() == B5.previous_hash:
        print ("Error! Couldn't detect tampering.")
    else:
        print ('Success! Tampering detected.')


"""
@Description: item2vec
@Author: 吕明伟
@Date: 2021-5-13
"""
from gensim.models import Word2Vec
import pandas as pd
import os
import pickle

def w2v(df, f, L=128):
    print("w2v",f)
    sentences=[]
    for line in df[f].values:
        sentences.append(line.split())
    print("Sentence Num {}".format(len(sentences)))
    w2v = Word2Vec(sentences, vector_size=L, window=20, min_count=1, sg=1, workers=8, epochs=10)
    print("save w2v to {}".format(os.path.join('data', f +".{}d".format(L))))
    pickle.dump(w2v,open(os.path.join('data', f + ".{}d".format(L)), 'wb'))

def get_list(x):
    return " ".join(list(x['MovieID']))

if __name__ == '__main__':

    path = 'data/ml-1m/'
    ratings_col = ['UserID', 'MovieID', 'Rating', 'Timestamp']
    ratings = pd.read_csv(f'{path}ratings.dat', sep="::", names=ratings_col)
    ratings['MovieID'] = ratings['MovieID'].astype(str)
    tmp = ratings[['UserID', 'MovieID']].groupby('UserID').apply(get_list)
    tmp = tmp.reset_index()
    tmp.columns = ['UserID', 'MovieID']
    w2v(tmp, 'MovieID', L=64)


import math
import pygame
from pygame.locals import *

import ScummPyUtils
import ScummPyAnimation
import Textify

class Character:

	def __init__(self, ResourcePath="resources/", CharacterName="bozo"):

		self.ResourcePath=ResourcePath

		#Read in Character properties
		charFP = open(self.ResourcePath + "characters/" + CharacterName + "/" + CharacterName + ".scd", "r")
		for line in charFP:
			if len(line)>0:
				words = line.split()
				if words[0] == 'StartX':
					self.x = int(words[1])
				if words[0] == 'StartY':
					self.y = int(words[1])
				if words[0] == 'Speed':
					self.Speed = int(words[1])
				if words[0] == 'Layer':
					self.Layer = int(words[1])
				if words[0] == 'Startroom':
					self.Startroom = words[1]
				if words[0] == 'WalkSteps':
					self.WalkSteps = int(words[1])
				if words[0] == 'WalkLeftRow':
					self.WalkLeftRow = int(words[1])
				if words[0] == 'WalkRightRow':
					self.WalkRightRow = int(words[1])
				if words[0] == 'WalkUpRow':
					self.WalkUpRow = int(words[1])
				if words[0] == 'WalkDownRow':
					self.WalkDownRow = int(words[1])
				if words[0] == 'SizeX':
					self.SizeX = int(words[1])
				if words[0] == 'SizeY':
					self.SizeY = int(words[1])
				if words[0] == 'FontFile':
					self.FontFile = words[1]
				if words[0] == 'TalkX':
					self.TalkX = int(words[1])
				if words[0] == 'TalkY':
					self.TalkY = int(words[1])

		charFP.close()

		self.scale=1
		self.xDest=self.x
		self.yDest=self.y
		self.pos=(int(self.x),int(self.y))

		self.xVel=0
		self.yVel=0

		self.SheetX=0
		self.SheetY=0
		self.WalkStep=0

		self.blitRect = pygame.Rect(0,0,self.SizeX,self.SizeY)

		#Masks and Sprite Sheet
		self.imPlayer =  pygame.image.load(self.ResourcePath + "characters/" + CharacterName + "/" + CharacterName + "Scale.png")
		self.imLayer = pygame.image.load(ResourcePath + "rooms/" + self.Startroom + "/room" + self.Startroom + "_layers.png")
		self.imWM = pygame.image.load(ResourcePath + "rooms/" + self.Startroom + "/room" + self.Startroom + "_walkmask.png")
		self.imSheet = pygame.image.load(self.ResourcePath + "characters/" + CharacterName + "/" + CharacterName + "Sheet.png")

		#This is a "hook" for allowing the main.py to attach animations to a character.
		#Display code will need to be updated to add it...
		self.Animations = dict()

		#Speaking setup
		self.TalkFont = Textify.BlitFont(ResourcePath, self.FontFile)
		self.SayString = ""
		self.TicsToTalk = 0

	def GoTo(self, destRoom, coords):
		self.imLayer = pygame.image.load(destRoom.ResourcePath + "rooms/" + destRoom.RoomName + "/room" + destRoom.RoomName + "_layers.png")
		self.imWM = pygame.image.load(destRoom.ResourcePath + "rooms/" + destRoom.RoomName + "/room" + destRoom.RoomName + "_walkmask.png")
		self.pos = coords
		self.x = coords[0]
		self.y = coords[1]
		self.xDest = coords[0]
		self.yDest = coords[1]

	def WalkTo(self, WalkX, WalkY):
		#print "X " + str(WalkX) + " Y " + str(WalkY) + " "

		#Set Destination
		self.xDest=WalkX
		self.yDest=WalkY

		#Set Velocity
		Mag=math.sqrt((self.xDest-self.x)*(self.xDest-self.x)+(self.yDest-self.y)*(self.yDest-self.y))
		self.xVel=-self.Speed*(self.x-self.xDest)/Mag
		self.yVel=-self.Speed*(self.y-self.yDest)/Mag
		#print ("X: "+str(self.x)+"Y:"+str(self.y))

	def Update(self):
		#print(self.x)

		Distance = ((self.xDest-self.x)*(self.xDest-self.x)+(self.yDest-self.y)*(self.yDest-self.y))
		MaskPix=self.imWM.get_at((int(self.x),int(self.y)))
		self.scale=float(MaskPix[0])/255
		if (Distance<25):
			#print("Stopping")
			self.xVel=0
			self.yVel=0
		else:
			self.x=self.xVel+self.x
			self.y=self.yVel+self.y
			MaskPix=self.imWM.get_at((int(self.x),int(self.y)))
			if(MaskPix[0]==0):
				self.x=-self.xVel+self.x
				self.y=-self.yVel+self.y
				self.xVel=0
				self.yVel=0

			#Animation
			self.WalkStep=(self.WalkStep+1)%self.WalkSteps

			if(self.yVel>3):
				self.SheetX = self.WalkStep*self.SizeX
				self.SheetY = self.WalkDownRow*self.SizeY
			if(self.yVel<-3):
				self.SheetX = self.WalkStep*self.SizeX
				self.SheetY = self.WalkUpRow*self.SizeY
			if(self.xVel>3):
				self.SheetX = self.WalkStep*self.SizeX
				self.SheetY = self.WalkRightRow*self.SizeY
			if(self.xVel<-3):
				self.SheetX = self.WalkStep*self.SizeX
				self.SheetY = self.WalkLeftRow*self.SizeY

		self.Layer=ScummPyUtils.GetIndex(self.imLayer.get_at((int(self.x),int(self.y))))
		self.pos=(int(self.x),int(self.y))

	def Display(self, imOutPut):
		#Spit to display
		self.imPlayer.fill((255,0,255))
		self.imPlayer.set_colorkey((255,0,255)) #probably inefficient, move to constructor if laggy?

		Middle = self.SizeX/2

		#Animation Handling
		AnyAnimationOn=False
		for k,Anim in self.Animations.iteritems():
			if Anim.Running:
				AnyAnimationOn=True
				Anim.Display(imOutPut,(0,0))

		if not AnyAnimationOn:
			self.imPlayer.blit(self.imSheet,(-self.SheetX,-self.SheetY))
			imOutPut.blit(pygame.transform.scale(self.imPlayer,(int(self.SizeX*self.scale),int(self.SizeY*self.scale))), (self.x-int(Middle*self.scale),self.y-int(self.SizeY*self.scale)))
			self.blitRect.left = self.SheetX
			self.blitRect.top = self.SheetY
			#imOutPut.blit(pygame.transform.scale(self.imSheet,(int(self.SizeX*self.scale),int(self.SizeY*self.scale))), (self.x-int(Middle*self.scale),self.y-int(self.SizeY*self.scale)),self.blitRect)

		if self.TicsToTalk > 0:
			self.TalkFont.BlitTextCenter(imOutPut,(self.pos[0]+self.TalkX,self.pos[1]+self.TalkY),self.SayString)
			self.TicsToTalk -= 1

	def SetWalkMask(self, newWalkMask):
		self.imWM.fill((0,0,0))
		self.imWM.blit(newWalkMask,(0,0))

	def Say(self, inputString = "!!!", duration = "15"):
		self.TicsToTalk = duration
		self.SayString = inputString


# -*- coding: utf-8 -*-
import torch
import numpy as np
from torch.utils.data import Dataset
from data_process import pickle_load_large_file

def collate(data):
    widsC, cidsC, widsQ, cidsQ, a_start, a_end, id, answerable = zip(*data)
    widsC = torch.Tensor(widsC).long()
    cidsC = torch.Tensor(cidsC).long()
    widsQ = torch.Tensor(widsQ).long()
    cidsQ = torch.Tensor(cidsQ).long()
    a_start = torch.Tensor(a_start).long()
    a_end = torch.Tensor(a_end).long()
    id = torch.Tensor(id).long()
    answerable = torch.Tensor(answerable).long()

    return  widsC, cidsC, widsQ, cidsQ, a_start, a_end, id, answerable

class SQuADDataset(Dataset):

    def __init__(self, data_path):
        self.data = pickle_load_large_file(data_path)
        self.num = len(self.data)

    def __len__(self):
        return self.num

    def __getitem__(self, item):
        return (self.data[item]['wids_context'],
                self.data[item]['cids_context'],
                self.data[item]['wids_question'],
                self.data[item]['cids_question'],
                self.data[item]['a_start'],
                self.data[item]['a_end'],
                self.data[item]['id'],
                self.data[item]['answerable'])


# -*- coding: utf-8 -*-

import re
import datetime
from core import execute_task, parse
from sslyze import SSLyzeLauncher

class SSLValidityTask(SSLyzeLauncher):
    """
    SSL validity checker
    """

    def _get_commands(self):
        """
        Returns the list of sslyze options
        """
        return [
            "--regular",
        ]

    def _parse_result(self, data):
        """
        Parses the sslyze output (@data)
        """
        def as_datetime(s):
            month, day, h, m, s, year = re.match(
                (r'([A-Z][a-z]{2})\s+(\d+) (\d\d):(\d\d):(\d\d) '
                r'(\d{4}) GMT'), s
            ).groups()

            month = ['JAN', 'FEB', 'MAR', 'APR', 'MAY', 'JUN',
                'JUL', 'AUG', 'SEP', 'OCT', 'NOV', 'DEC'].index(month.upper()) + 1

            return datetime.datetime(
                int(year), month, int(day),
                int(h), int(m), int(s)
            )

        parser = parse.LineByLineParser()
        parser['Not Before:': as_datetime] = 'from'
        parser['Not After:': as_datetime] = 'to'
        parser['Certificate is Trusted'] = 'trusted'

        try:
            data = parser.parse(data.split('\n'))
        except ValueError:
            return "Certificate is not valid."

        if 'trusted' in data and data['from'] <= datetime.datetime.now() <= data['to']:
            return 'Certificate is valid (expires %s)' % data['to']
        else:
            return 'Certificate is not valid (expires %s)' % data['to']

    def test(self):
        """
        Test function
        """
        self.host = "www.google.com"
        self.main()

execute_task(SSLValidityTask)


from Gene_Network import Gene_Network
from data_structures import Gene
import itertools
import os
import json
import matplotlib.pyplot as plt
import numpy as np

path = "C:/Users/Lenovo/PycharmProjects/INDEP/cell_collective_networks_2018"
big_entries= ["CC_CD4p_Tcell_Diff",
"CC_Differentiation_T_lymphocytes",
"CC_Inflammatory_Bowel_Disease_IBD_Model",
"CC_Glucose_Repression_Signaling_2009",
"CC_HCC1954_Breast_Cell_Line_Long-term_ErbB_Network",
"CC_IGVH_mutations",
"CC_Lymphopoiesis_Regulatory_Network",
"CC_Signaling_in_Macrophage_Activation",
"CC_Signal_Transduction_in_Fibroblasts",
"CC_Treatment_of_Castration_Resistant_Prostate_Cancer",
"CC_Yeast_Apoptosis"]
entries = os.listdir(path)

distributions_and_sensitivities={}
truths_and_sensitivities={}
input_numbers=[2,3,4,5,6]
d=[0,0]
path="C:/Users/Lenovo/PycharmProjects/INDEP/network_data_altered"
for input_number in  input_numbers:
    truths=[0]*(2**(input_number))
    for i in range(0,len(entries)):
        cell=entries[i]
        if cell in big_entries:
            continue
        expressions = path + "/" + cell + "expressions.ALL.txt"
        externals = path + "/" + cell + "external_components.ALL.txt"
        gn = Gene_Network(cell, expressions, externals)
        #gn.sensitivity_test()
        for gene in gn.genes:
            if len(gene.is_influenced)+len(gene.is_influenced_externals)!=input_number:
                continue
           # temp,classes,R=gene.find_symmetries()
            truth=gene.activity_bias()*(2**input_number)
            truths[int(truth)] +=1

           # if temp in distributions_and_sensitivities:
            #    a= list(distributions_and_sensitivities[temp])
             #   a.append(gene.sensitivity)
              #  a=tuple(a)
               # distributions_and_sensitivities[temp]=a
            #else:
             #   distributions_and_sensitivities[temp]=(gene.sensitivity,gene.sensitivity)

            if truth in truths_and_sensitivities:
                a= list(truths_and_sensitivities[truth])
                a.append(gene.sensitivity)
                a=tuple(a)
                truths_and_sensitivities[truth]=a
            else:
                truths_and_sensitivities[truth]=(gene.sensitivity,gene.sensitivity)
    plt.figure()
    i=0
    #for distribution in distributions_and_sensitivities:
     #   for point in distributions_and_sensitivities[distribution]:
      #      plt.plot(distribution,point,'*')
       # i +=1

    plt.figure()
    i=0
    #for distribution in truths_and_sensitivities:
     #   for point in truths_and_sensitivities[distribution]:
      #      plt.plot(distribution,point,'*k')
       # i +=1
    plt.show()

    plt.figure()
    for i in range(len(truths)):
        plt.plot(i,truths[i],'*k')
        if i%2==0:
            d[0]+=truths[i]
        else:
            d[1] += truths[i]
    plt.grid()
    plt.xlabel('Number of Truths')
    plt.ylabel('Number of genes')
    plt.title('Number of Truths in Full Logical Table for '+str(input_number)+'-input')
    plt.show()

plt.bar(['even','odd'], d, width=0.8, align='center', )
plt.title('Number of Truths for all functions')
plt.savefig('number_of_truths_all')
plt.show()

#with open('distributions_and_sensitivities.json', 'w') as fp:
 #   json.dump(distributions_and_sensitivities, fp)
#with open('truths_and_sensitivities.json', 'w') as fp:
 #   json.dump(truths_and_sensitivities, fp)

plots/Number of truths plots/number_of_truths_analysis.py


#!/usr/bin/env python3

import json
from py.src.time_util import (
    convert_dt_to_seconds,
)
from py.src.settings import (
    ENVARS,
)

wd_cookie = ENVARS.cookie
_TEMP_COOKIE_FILE = 'temp/cookie_response.json'

class CookieData():

    def __init__(self, updated_at_seconds, response_text):
        self.updated_at_seconds = updated_at_seconds
        self.response_text = response_text

    def to_file(self, cookie_file):
        json.dump(self.__dict__, cookie_file)

    def is_old(self):
        return (
            300 < CookieData.utcnow_seconds() - self.updated_at_seconds
        )

    @property
    def is_invalid(self):
        return (
            "Session Expired" in self.response_text or
            "session error" in self.response_text
        )

    @staticmethod
    def from_file(cookie_file):
        json_data = json.load(
            cookie_file,
        )
        return CookieData(
            json_data['updated_at_seconds'],
            json_data['response_text'],
        )

    @staticmethod
    def utcnow_seconds():
        return convert_dt_to_seconds()

    @staticmethod
    def from_response(response):
        return CookieData(
            CookieData.utcnow_seconds(),
            response.text,
        )

def load_cookie_status():
    try:
        with open(_TEMP_COOKIE_FILE) as f:
            return CookieData.from_file(f)
    except FileNotFoundError:
        pass
    return None

def save_cookie_status(response):
    cd = CookieData.from_response(response)
    with open(_TEMP_COOKIE_FILE, 'w') as f:
        cd.to_file(f)
    return cd


from django.urls import path
from products.views import ProductView, CategoriesView, ProductCreateView, DetailProductView, MainView

urlpatterns = [
    path('product/', ProductView.as_view()),
    path('categories/', CategoriesView.as_view()),
    path('online-store/', MainView.as_view()),
    path('product/<int:id>/', DetailProductView.as_view()),
    path('product/create/', ProductCreateView.as_view()),

]


import xml.etree.ElementTree as ET
import re
import pandas as pd
from bs4 import BeautifulSoup as bs

def parse_patents(file_name):

    # open file in read mode
    file = open(file_name, "r")
    file_content_raw = file.read()
    file.close()
    text1 = re.compile("<\?xml version\=\"1\.0\" encoding\=\"UTF\-8\"\?>")
    file_content = text1.split(file_content_raw)
    while "" in file_content:
        file_content.remove('')

    dic = {
    'juridiction' : [],
    'pub_year' : [],
    'kind' : [],
    'app_year' : [],
    'app_type' : [],
    'prior_year' : [],
    'prior_country' : [],
    'title' : [],
    'assignee' : [],
    'inventors' : [],
    'abstract' : [],
    'claims' : []
    }

    for i, file in enumerate(file_content):
    #i = file_content[10]
        root = ET.fromstring(str(bs(file, 'xml')))
        if root.tag == 'us-patent-application':
            # getting country of publication from attributes of the roots
            dic['juridiction'].append(root.attrib['country'])

            # giving name to childs of root
            bibliographic_data_elem = root.find('us-bibliographic-data-application')
            abstract = root.find('abstract')
            #drawings = root.find('drawings')
            #description = root.find('description')
            claims = root.find("claims")

            # appending abstract to the dic
            dic['abstract'].append(abstract.find('p').text)

            # appending publication year and kind
            for i in bibliographic_data_elem.find('publication-reference').find('document-id').getchildren():
                if i.tag == 'date':
                    yyyy = i.text[0:4]
                    # mm = i.text[4:6]
                    # dd = i.text[6:8]
                    dic['pub_year'].append(int(yyyy))
                if i.tag == 'kind':
                    dic['kind'].append(i.text)

            # appending application year and application utility
            for i in bibliographic_data_elem.find('application-reference').find('document-id').getchildren():
                if i.tag == 'date':
                    yyyy = i.text[0:4]
                    # mm = i.text[4:6]
                    # dd = i.text[6:8]
                    dic['app_year'].append(int(yyyy))
            dic['app_type'].append(bibliographic_data_elem.find('application-reference').attrib['appl-type'])

            # for priority claims/ if published in other countries
            priority_claims = bibliographic_data_elem.find('priority-claims')
            if priority_claims != None:
                for i in priority_claims.find('priority-claim').getchildren():
                    if i.tag == 'country':
                        dic['prior_country'].append(i.text)
                    if i.tag == 'date':
                        yyyy = i.text[0:4]
                        dic['prior_year'].append(int(yyyy))
                        #mm = i.text[4:6]
                        #dd = i.text[6:8]
            else:
                dic['prior_country'].append('')
                dic['prior_year'].append('')

            # adding invention title
            dic['title'].append(bibliographic_data_elem.find('invention-title').text)

            # adding assignees
            assignees = bibliographic_data_elem.find('assignees')
            if assignees != None:
                org_names = ''
                for assignee in assignees:
                    if assignee.find('orgname') != None:
                        org_name = assignee.find('orgname').text
                        org_names = org_names + org_name + ';;'
                dic['assignee'].append(org_names)
            else:
                dic['assignee'].append(None)

            # adding Inventors
            us_parties = bibliographic_data_elem.find('us-parties')
            inventors = us_parties.find('inventors').getchildren()
            inventor = ''
            #countries = []
            addressbooks = []
            for i in inventors:
                addressbooks.append(i.getchildren()[0])
            for i in addressbooks:
                name = ''
                childs = i.getchildren()
                for child in childs:
                    if child.tag == 'last-name':
                        name += child.text
                    if child.tag == 'first-name':
                        name = child.text + ' ' + name
                    #if child.tag == 'address':
                    #    countries.append(child.find('country').text)
                inventor = inventor + name + ';;'
            dic['inventors'].append(inventor)

           # adding claims
            claims_child = claims.getchildren()
            claims = ''
            for childs in claims_child:
                for child in childs:
                    for i in child:
                        if i.tag == 'claim-text':
                            if i.text != None:
                                claims += i.text + ' '
            dic['claims'].append(claims)

            # feild if invention
            # backgroung
            # details of drawing

    df = pd.DataFrame(dic)
    return df


from django.contrib.auth.models import User
from django.shortcuts import render,redirect, get_object_or_404
from django.http import Http404,HttpResponseRedirect
from django.core.exceptions import ObjectDoesNotExist
from django.contrib.auth.forms import UserCreationForm
from instagram.forms import CommentsForm,PostForm, Registration, UpdateProfile, UpdateUser
from instagram.models import Follows, Image, Likes, Profile
from django.contrib import messages
from django.contrib.auth.decorators import login_required

# Create your views here.
@login_required(login_url='/accounts/login/')
def home(request):
    comment_form = CommentsForm()
    post_form = PostForm()
    images = Image.display_images()
    all_users = User.objects.all()
    base_image = "https://res.cloudinary.com/dtbko4o5h/image/upload/v1638685260/"
    return render(request, 'home.html',{"images":images,"comment_form":comment_form,"post":post_form,"base_image":base_image,"all_users":all_users})

@login_required(login_url='/accounts/login/')
def post_image(request):
    if request.method == 'POST':
        post_form = PostForm(request.POST,request.FILES)
        if post_form.is_valid():
            the_post = post_form.save(commit = False)
            the_post.user = request.user
            the_post.save()
        return redirect('home')
    else:
        post_form = PostForm()
    return render(request,'post.html',{"post_form":post_form})

def detail(request,image_id):
    current_user = request.user
    try:
        image = get_object_or_404(Image, pk = image_id)
    except ObjectDoesNotExist:
        raise Http404()
    return render(request, 'imageDetails.html', {'image':image,'current_user':current_user})

@login_required(login_url='/accounts/login/')
def search_results(request):
    if 'search_user' in request.GET and request.GET["search_user"]:
        search_term = request.GET.get('search_user')
        users = Profile.search_profiles(search_term)
        images = Image.search_images(search_term)
        return render(request,'search.html',{"users":users,"images":images})
    else:
        return render(request,'search.html')

def register(request):
    if request.method == 'POST':
        form = Registration(request.POST)
        if form.is_valid():
            form.save()
            email = form.cleaned_data['email']
            username = form.cleaned_data.get('username')

            messages.success(request,f'Account for {username} created,you can now login')
        return redirect('login')
    else:
        form = Registration()
    return render(request,'registration/registration_form.html',{"form":form})

@login_required(login_url='/accounts/login/')
def profile(request):
    comment_form = CommentsForm()
    current_user = request.user
    images = Image.objects.all().order_by('-pub_date')
    all_users = User.objects.all()
    user_images = Image.objects.filter(user_id = current_user.id).all()

    return render(request,'profile.html',{"images":images,'all_users':all_users,'comment_form':comment_form,'user_images':user_images,"current_user":current_user})

@login_required(login_url='/accounts/login/')
def users_profile(request,pk):
    comment_form = CommentsForm()
    user = User.objects.get(id = pk)
    image = Image.objects.filter(user = user)
    c_user = request.user
    return render(request,'users_profile.html',{"user":user,'comment_form':comment_form,"image":image,"c_user":c_user})

@login_required(login_url='/accounts/login/')
def profile_update(request):
    if request.method == 'POST':
        user_form = UpdateUser(request.POST,instance=request.user)
        profile_form = UpdateProfile(request.POST,request.FILES,instance=request.user.profile)
        if user_form.is_valid() and profile_form.is_valid():
            user_form.save()
            profile_form.save()
            messages.success(request,'Your Profile account has been updated successfully')
        return redirect('profile')
    else:
        user_form = UpdateUser(instance=request.user)
        profile_form = UpdateProfile(instance=request.user.profile)
    params = {
        'user_form':user_form,
        'profile_form':profile_form
    }
    return render(request,'update_profile.html',params)

@login_required(login_url='/accounts/login/')
def delete(request,image_id):
    current_user = request.user
    image = Image.objects.get(pk=image_id)
    if image:
        image.delete_post()
    return redirect('home')

@login_required(login_url='/accounts/login/')
def comment(request,image_id):
    current_user = request.user
    image = Image.get_single_image(id=image_id)
    if request.method == 'POST':
        comment_form = CommentsForm(request.POST)

        if comment_form.is_valid():
            comment = comment_form.save(commit=False)
            comment.user = current_user
            comment.image_id = image_id
            comment.save()
        return redirect('/')

    else:
        comment_form = CommentsForm()
        return render(request,'comment.html',{"comment_form":comment_form,"image":image})

@login_required(login_url='/accounts/login/')
def allcomments(request,image_id):
    image = Image.objects.filter(pk = image_id).first()
    return render(request,'comments.html',{"image":image})

@login_required(login_url='/accounts/login/')
def follow(request,user_id):
    followee = request.user
    followed = Follows.objects.get(pk=user_id)
    follow_data,created = Follows.objects.get_or_create(follower = followee,followee = followed)
    follow_data.save()
    return redirect('others_profile')

# def like(request, image_id):
#     current_user = request.user
#     image=Image.objects.get(id=image_id)
#     new_like,created= Likes.objects.get_or_create(liker=current_user, image=image)
#     new_like.save()

#     return HttpResponseRedirect(request.META['HTTP_REFERER'])

def image_likes(request,image_id):
    image =  Image.get_single_image(image_id)
    user = request.user
    user_id = user.id

    if user.is_authenticated:

        image.save()

    return redirect('home')

@login_required(login_url='/accounts/login/')
def unfollow(request,user_id):
    followee = request.user
    follower = Follows.objects.get(pk=user_id)
    follow_data = Follows.objects.filter(follower = follower,followee = followee).first()
    follow_data.delete()
    return redirect('users_profile')


# -*- coding: utf-8 -*-
# @Time    : 2019/7/5 12:24
# @Author  : Ze-peng Wu(午泽鹏)
# @Email   : wuzepeng_sxu@126.com
# @File    : trainEmb.py
# @Software: PyCharm

from gensim.models.word2vec import Word2Vec
from gensim.models.word2vec import LineSentence
import numpy as np
import warnings

warnings.filterwarnings(action='ignore', category=UserWarning, module='gensim')
def trainW2v():
    sentences = LineSentence('../datagrand/newCropus.txt')
    model = Word2Vec(sentences, size=300, window=5, min_count=1, sg=1, workers=4)  # sg=0 使用cbow训练, sg=1对低频词较为敏感
    model.save('../datagrand/w2v/grand.w2v.300d.txt')

def build_word_dict(model_path):
    """  获取word2vec模型的所有词向量:
    param model_path: 已经训练好的word2vec模型保存路径
    return:返回词向量词典
    """
    model = Word2Vec.load(model_path)
    vocab = model.wv.vocab
    word_vector = {}
    for word in vocab:
        # print (word)
        word_vector[word] = np.asarray(model[word],dtype='float32')
    return word_vector

if __name__ == '__main__':
    word_vector = build_word_dict('../datagrand/w2v/grand.w2v.300d.txt')
    print ('---')
    print (len(word_vector))
    print (word_vector['20000'])
    # trainW2v()


import sys
import random
import enum
from typing import List, Tuple
import psutil
import cv2
import os
import numpy as np
import torch.nn as nn
import time
from PyQt5.QtWidgets import *
from PyQt5.QtGui import QPixmap
from sklearn.decomposition import PCA
from sklearn.metrics import mean_squared_error
from torch.utils.data import Dataset
from torchvision import transforms
from torchvision import datasets

import matplotlib
import matplotlib.pyplot as plt
from PIL import Image
import torch
import torchvision
import torchvision.transforms as transforms
import torchsummary

img = None
video = None

def load_img():
    img_path, _ = QFileDialog.getOpenFileName(None, "Open File", r"D:\OneDrive - gs.ncku.edu.tw\oneDrive_NCKU_master\1st_semester\ComputerVision_and_Deeplearning\hw\hw2\Dataset_OpenCvDl_Hw2", "Image Files (*.jpg *.jpeg *.png *.bmp)", None)
    global img
    img = cv2.imread(img_path)

def load_video():
    video_path, _ = QFileDialog.getOpenFileName(None, 'Open File', r"D:\OneDrive - gs.ncku.edu.tw\oneDrive_NCKU_master\1st_semester\ComputerVision_and_Deeplearning\hw\hw2\Dataset_CvDl_Hw2", "Video Files (*.mp4 *.avi);;All Files (*)", None)
    global video
    video = cv2.VideoCapture(video_path)

def resize_show(string, img, width, height):
    shape_ratio = min(600/img.shape[1], 400/img.shape[0])
    resized_img = cv2.resize(img, None, fx=shape_ratio, fy=shape_ratio)
    cv2.imshow(string,resized_img)
    cv2.moveWindow(string, width, height)#通常是 (50,50)，若要左右比較可改成 (450, 50)
    if cv2.waitKey(1000) != -1:
        cv2.destroyAllWindows()

#Background_Subtraction
class Assign_1:
    def Background_Subtraction(self):
        # Create a background subtractor
        subtractor = cv2.createBackgroundSubtractorKNN(history=500, dist2Threshold=400, detectShadows=True)

        global video
        while True:
            ret, frame = video.read()

            if not ret:
                break

            # Blur the frame
            blurred_frame = cv2.GaussianBlur(frame, (5, 5), 0)

            # Apply background subtractor to get the mask
            mask = subtractor.apply(blurred_frame)

            # Create a side-by-side display of the original frame, foreground mask, and backgound mask
            display_frame = np.hstack((frame, cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR), cv2.bitwise_and(frame, frame, mask=mask)))

            # Display the result
            cv2.imshow("Result", display_frame)

            # Break the loop if 'Esc' key is pressed
            if cv2.waitKey(30) == 27:
                break

        video.release()
        cv2.destroyAllWindows()

#Optical flow
class Assign_2:
    def __init__(self):
        self.old_frame = None
        self.old_gray  = None
        self.nose_point = None
        self.canvas = None

    def Preprocessing(self):
        video = cv2.VideoCapture(r"D:\OneDrive - gs.ncku.edu.tw\oneDrive_NCKU_master\1st_semester\ComputerVision_and_Deeplearning\hw\hw2\Dataset_CvDl_Hw2\Q2\optical_flow.mp4")
        # Read the first frame
        ret, self.old_frame = video.read()
        if ret:
            # Convert the frame to grayscale
            self.old_gray  = cv2.cvtColor(self.old_frame, cv2.COLOR_BGR2GRAY)

            # Use cv2.goodFeaturesToTrack to detect the point at the bottom of the doll's nose
            self.nose_point = cv2.goodFeaturesToTrack(self.old_gray, maxCorners=1, qualityLevel=0.3, minDistance=7, blockSize=7)

            if self.nose_point is not None:
                # Extract the coordinates of the detected point
                x, y = self.nose_point[0].ravel()

                # Draw a red cross mark at the detected point
                cv2.line(self.old_frame, (int(x) - 10, int(y)), (int(x) + 10, int(y)), (0, 0, 255), 4)
                cv2.line(self.old_frame, (int(x), int(y) - 10), (int(x), int(y) + 10), (0, 0, 255), 4)

                # Display the frame using cv2.imshow
                resize_show('Frame with Nose Point', self.old_frame, 50, 50)
                cv2.waitKey(0)
                cv2.destroyAllWindows()

    def Video_tracking(self):
        video = cv2.VideoCapture(r"D:\OneDrive - gs.ncku.edu.tw\oneDrive_NCKU_master\1st_semester\ComputerVision_and_Deeplearning\hw\hw2\Dataset_CvDl_Hw2\Q2\optical_flow.mp4")
        # set min size of tracked object, e.g. 15x15px
        parameter_lucas_kanade = dict(winSize=(15, 15), maxLevel=2, criteria=(cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03))
        # Create a mask image for drawing purposes
        mask = np.zeros_like(self.old_frame)
        # Create some random colors
        color = np.random.randint(0,255,(100,3))

        while True:
            # get next frame
            ok, frame = video.read()
            if not ok:
                print("[INFO] end of file reached")
                break

            # prepare grayscale image
            frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

            # update object corners by comparing with found self.nose_point in initial frame
            update_nose_point, status, errors = cv2.calcOpticalFlowPyrLK(
                self.old_gray,
                frame_gray,
                self.nose_point,
                None,
                **parameter_lucas_kanade,
            )

            # only update self.nose_point if algorithm successfully tracked
            new_nose_point = update_nose_point[status == 1]
            # to calculate directional flow we need to compare with previous position
            old_nose_point = self.nose_point[status == 1]

            for i, (new, old) in enumerate(zip(new_nose_point, old_nose_point)):
                a, b = new.ravel()
                c, d = old.ravel()

                # draw line between old and new corner point with random colour
                mask = cv2.line(mask, (int(a), int(b)), (int(c), int(d)), color[i].tolist(), 2)
                # draw circle around new position
                frame = cv2.circle(frame, (int(a), int(b)), 5, color[i].tolist(), -1)
            result = cv2.add(frame, mask)

            cv2.imshow('Optical Flow', result)
            k = cv2.waitKey(30) & 0xff
            if k == 27:
                break
            # overwrite initial frame with current before restarting the loop
            self.old_gray = frame_gray.copy()
            # update to new edges before restarting the loop
            self.nose_point = new_nose_point.reshape(-1, 1, 2)

        video.release()
        cv2.destroyAllWindows()

#PCA – Dimension Reduction
class Assign_3:
    def pca(self):
        img = cv2.imread(r"D:\OneDrive - gs.ncku.edu.tw\oneDrive_NCKU_master\1st_semester\ComputerVision_and_Deeplearning\hw\hw2\Dataset_CvDl_Hw2\Q3\logo.jpg")
        gray_image  = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

        # Normalize gray scale image
        normalized_gray_image = np.float32(gray_image) / 255.0
        # flat_image = normalized_gray_image.flatten()
        width, high = normalized_gray_image.shape
        min_components_number = None

        for n in range(1, 350):
            # Apply PCA
            pca = PCA(n_components=n)
            # reduced_image = pca.fit_transform(flat_image.reshape(1, -1))
            reduced_image = pca.fit_transform(normalized_gray_image)
            reconstructed_image = pca.inverse_transform(reduced_image)

            # Calculate Mean Squared Error
            mse = 0.0
            for i in range(width):
                for j in range(high):
                    mse += (normalized_gray_image[i][j]-reconstructed_image[i][j])**2
            # mse = mse**0.5
            # mse = mse / (width*high)

            # Check if the error is less than or equal to the threshold
            if mse <= 3.0:
                print(mse)
                print(f"n value = {n}")
                min_components_number = n
                break

        # Reconstruct the image using the selected components
        pca = PCA(n_components=min_components_number)
        reduced_image = pca.fit_transform(normalized_gray_image)
        reconstructed_image = pca.inverse_transform(reduced_image)

        # Reshape the reconstructed image
        reconstructed_image = reconstructed_image.reshape(gray_image.shape)

        # Plot the original and reconstructed images
        resize_show("Original Image", img, 50, 50)
        resize_show("gray Image", gray_image, 450, 50)
        resize_show(f"Reconstructed Image(n={min_components_number})", reconstructed_image, 850, 50)

class Assign_4:
    def show_model(self):
        model = torchvision.models.vgg19_bn()
        torchsummary.summary(model, (3, 224, 224))

    def show_acc_and_loss(self):
        img = cv2.imread("./assign_4_cureve.png")
        if img is not None:
            cv2.imshow("accuracy and loss", img)
            cv2.moveWindow("accuracy and loss", 50 , 50)#通常是 (50,50)，若要左右比較可改成 (450, 50)
            if cv2.waitKey(1000) != -1:
                cv2.destroyAllWindows()
        else:
            print("Error: Failed to read the image.")

    def predict(self):
        # 定義圖片轉換
        transform = transforms.Compose([
            transforms.Grayscale(num_output_channels=1),
            transforms.ToTensor(),
        ])

        img = Image.open("./test.png")
        test_img = transform(img).unsqueeze(0)
        model = torch.load('./assign_4_model.pth')
        model.eval()
        # 進行模型推論
        with torch.no_grad():
            output = model(test_img)

        # 獲取預測標籤
        _, predicted_label = torch.max(output, 1)
        _ , index = torch.max(output, 1) # 通过argmax方法，得到概率最大的处所对应的索引
        # 模型預測的概率分佈
        probabilities = torch.nn.functional.softmax(output, dim=1)[0]
        class_probabilities = np.sum(probabilities.numpy().reshape(1, -1), axis=1)

        img_classes = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '0']

        x_labels = img_classes
        plt.bar(np.array(img_classes), class_probabilities)
        plt.xlabel('Class')
        plt.ylabel('Probability')
        plt.title('Probability of each class')
        plt.xticks(rotation=45, ha='right') #調整標籤顯示方式，讓x軸上的標籤更容易閱讀。
        plt.tight_layout()

        # 顯示直方圖
        plt.show()

# 定義推論數據集類別
class InferenceDataset(Dataset):
    def __init__(self, root_dir, transform=None):
        self.dataset = datasets.ImageFolder(root_dir, transform=transform)

    def __len__(self):
        return len(self.dataset)

    def __getitem__(self, idx):
        return self.dataset[idx]

class Assign_5:
    def __init__(self):
        self.inference_dataset = None

    def load_img(self):
        # 設定推論數據集的路徑
        inference_path = r'D:\OneDrive - gs.ncku.edu.tw\oneDrive_NCKU_master\1st_semester\ComputerVision_and_Deeplearning\hw\hw2\Dataset_CvDl_Hw2\Q5_dataset\inference_dataset'

        # 定義轉換
        transform = transforms.Compose([
            transforms.Resize((224, 224)),
        ])

        # 創建推論數據集
        self.inference_dataset = InferenceDataset(inference_path, transform=None)

    def show_imgs(self):
        # 顯示一個來自每個類別的圖像
        class_labels = self.inference_dataset.dataset.classes
        fig, axs = plt.subplots(1, len(class_labels))

        for i, label in enumerate(class_labels):
            # 找到該類別的所有圖像索引
            indices = [index for index, label in enumerate(self.inference_dataset.dataset.targets) if label == i]

            # 從該類別的所有圖像中隨機選擇一個
            random_index = random.choice(indices)

            # 獲取圖像路徑
            img_path = self.inference_dataset.dataset.samples[random_index][0]
            img = plt.imread(img_path)

            # 顯示圖像
            axs[i].imshow(img, extent=[0, 224, 0, 224])
            axs[i].set_title(label)
            axs[i].axis('off')

        plt.show()

    def show_model(self):
        model = torchvision.models.resnet50(num_classes=2)
        model.fc = nn.Sequential(
            nn.Linear(2048, 1),
            nn.Sigmoid()
        )
        torchsummary.summary(model, (3, 224, 224))

    def show_comparison(self):
        pass


import tensorflow as tf
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
from keras.callbacks import ReduceLROnPlateau
from sklearn.ensemble import RandomForestRegressor
from sklearn.multioutput import MultiOutputRegressor
from sklearn.neighbors import KNeighborsRegressor
import xgboost as xgb
from xgboost import XGBRegressor
import pickle

df = pd.read_csv("/home/rakin/ML_Projects/new.csv")
df = df.drop(['Unnamed: 0'],axis=1)

Y = df.values[:,-3:] # Target
X = df.values[:,:-3] # Features

X_train, X_test, Y_train, Y_test = train_test_split(X,Y,test_size=0.3,shuffle=True)

# Normalize the target variables using MinMaxScaler
scaler = MinMaxScaler()
scaler2 = MinMaxScaler()
target1 = scaler2.fit_transform(Y_train[:,0].reshape(-1,1))  #
target2 = scaler.fit_transform(Y_train[:,1].reshape(-1,1))
target3 = scaler.fit_transform(Y_train[:,2].reshape(-1,1))

# Concatenate the normalized target variables

Y_train = np.concatenate((target1, target2, target3), axis=1)

from keras.callbacks import *

# filepath='./val_aux_output_accuracy:{val_accuracy:.3f}-epochs:{epoch:03d}.hdf5'
filepath = './Best_model_6.hdf5'
#earlyStopping = EarlyStopping(monitor='val_aux_output_loss', patience=10, verbose=0, mode='min')
#mcp_save = ModelCheckpoint('.mdl_wts.hdf5', save_best_only=True, monitor='val_loss', mode='min')
checkpoint = ModelCheckpoint(filepath, monitor='val_mse', verbose = 1, save_best_only=True, mode='min')
reduce_lr = ReduceLROnPlateau(monitor='val_mse', factor=0.5, mode='min',patience=2,verbose=1,min_lr=0.0001)
#callbacks_list = [checkpoint]
callbacks_list = [checkpoint,reduce_lr]

# ## Modify This cell - Try USing XGBoosRegressor, RandomForestRegressor, CatBoostRegressor

model=tf.keras.Sequential([tf.keras.layers.BatchNormalization(),
                           tf.keras.layers.Dense(12,input_dim=12,activation='relu'),
                           tf.keras.layers.Dense(64,activation='relu'),
                           tf.keras.layers.Dense(128,activation='relu'),
                           tf.keras.layers.Dense(128,activation='relu'),
                           tf.keras.layers.Dense(512,activation='relu'),
                           tf.keras.layers.Dense(24,activation='relu'),
                           tf.keras.layers.Dense(16,activation='relu'),
                           tf.keras.layers.Dense(3,activation='linear')
                           ])
model.compile(optimizer='RMSprop',loss='mse',metrics=['mse'])
 #This is the new change
history = model.fit(X_train,Y_train,epochs=300,batch_size=64,validation_split=0.3,callbacks=callbacks_list)

plt.figure(figsize=(10,6))
training_accuracy=history.history['loss'][50:]
epochs=range(51,301)
vali_ac=history.history['val_loss'][50:]
plt.plot(epochs,training_accuracy,'r',label='Training MSE')
plt.plot(epochs,vali_ac,'g',label='Validation MSE')
plt.legend()
plt.show()
model.summary()

def error_calculate(yhat,ytrue):
    error1 = 0
    error2 = 0
    error3 = 0
    for i in range(len(yhat)):
        error1 = error1 + ((ytrue[i,0]-yhat[i,0])**2)
        error2 = error2 +  ((ytrue[i,1]-yhat[i,1])**2)
        error3 = error3 + ((ytrue[i,2]-yhat[i,2])**2)
    print("MSE:",error1/len(yhat)," ",error2/len(yhat)," ",error3/len(yhat))
    return error1/len(yhat),error2/len(yhat),error3/len(yhat)
# Model = tf.keras.models.load_model(filepath)
# yhat = Model.predict(X_test)
# y1=scaler2.inverse_transform(yhat[:,0].reshape(-1,1))
# y2=scaler.inverse_transform(yhat[:,1:].reshape(-1,2))
# yhat = np.concatenate([y1,y2],axis=1)
# ytrue = Y_test
# error_calculate(yhat,ytrue)

# Making DataFrame

# column = ['ROCOF(Predicted)','ROCOF(Actual)','Settling Frequency(Predicted)','Settling Frequency(Actual)','Nadir(Predicted)','Nadir(Actual)']
# result_df = pd.DataFrame(columns=column)
# result_df['ROCOF(Predicted)']= yhat[:,0]
# result_df['Settling Frequency(Predicted)']=yhat[:,1]
# result_df['Nadir(Predicted)']=yhat[:,2]
# result_df['ROCOF(Actual)']= ytrue[:,0]
# result_df['Settling Frequency(Actual)']=ytrue[:,1]
# result_df['Nadir(Actual)']=ytrue[:,2]
# result_df.to_csv("test_result.csv")

#KNNRegressor
knn = KNeighborsRegressor(n_neighbors=5,weights="distance",algorithm="auto",leaf_size=20,p=2)
model_KNN = MultiOutputRegressor(knn)
model_KNN.fit(X_train,Y_train)

#RF Regressor
rf = RandomForestRegressor(n_estimators = 10000,max_depth=10,min_samples_split=2,min_samples_leaf=1,max_features='sqrt', random_state = 42)
model_RF = MultiOutputRegressor(rf)
# model.compile(optimizer='RMSprop',loss='mse',metrics=['mse'])
model_RF.fit(X_train,Y_train)
#XGB Regressor
xgb = XGBRegressor(
                n_estimators=100,
                max_depth=3,
                learning_rate=0.1,
                subsample=0.8,
                colsample_bytree=0.8,
                alpha=0.1,
                reg_lambda = 0.1,
                objective='reg:squarederror')
model_XGB = MultiOutputRegressor(xgb)
model_XGB.fit(X_train,Y_train)

# Saving ML models
# file name, I'm using *.pickle as a file extension
filename_RF = "random_forest.pickle"
filename_KNN = "knn.pickle"
filename_XGB = "xgb.pickle"

# save model
pickle.dump(model_RF, open(filename_RF, "wb"))
pickle.dump(model_KNN, open(filename_KNN, "wb"))
pickle.dump(model_XGB, open(filename_XGB, "wb"))

# ### Ensembling 3 Models and Inserting To a Neural Network

# filepath='./val_aux_output_accuracy:{val_accuracy:.3f}-epochs:{epoch:03d}.hdf5'
filepath_ens = './Best_model_ensemble.hdf5'
#earlyStopping = EarlyStopping(monitor='val_aux_output_loss', patience=10, verbose=0, mode='min')
#mcp_save = ModelCheckpoint('.mdl_wts.hdf5', save_best_only=True, monitor='val_loss', mode='min')
checkpoint = ModelCheckpoint(filepath_ens, monitor='val_mse', verbose = 1, save_best_only=True, mode='min')
reduce_lr = ReduceLROnPlateau(monitor='val_mse', factor=0.5, mode='min',patience=2,verbose=1,min_lr=0.0001)
#callbacks_list = [checkpoint]
callbacks_list = [checkpoint,reduce_lr]
def ensemble_training(model_RF,model_KNN,model_XGB,X_train,Y_train):
    y1 = model_RF.predict(X_train)
    y2 = model_KNN.predict(X_train)
    y3 = model_XGB.predict(X_train)
    Y_tr = np.concatenate([y1,y2,y3],axis=1)
    print(Y_tr.shape)
    model=tf.keras.Sequential([tf.keras.layers.Dense(12,input_dim=9,activation='relu'),
                           tf.keras.layers.Dense(64,activation='relu'),
                           tf.keras.layers.Dense(128,activation='relu'),
                           tf.keras.layers.Dense(128,activation='relu'),
                           tf.keras.layers.Dense(512,activation='relu'),
                           tf.keras.layers.Dense(512,activation='relu'),
                           tf.keras.layers.Dense(265,activation='relu'),
                           tf.keras.layers.Dense(24,activation='relu'),
                           tf.keras.layers.Dense(16,activation='relu'),
                           tf.keras.layers.Dense(3,activation='linear')
                           ])

    model.compile(optimizer='RMSprop',loss='mse',metrics=['mse'])

    history = model.fit(Y_tr,Y_train,epochs=300,batch_size=32,validation_split=0.3,callbacks=callbacks_list)
    model.summary()
    return history, model

history, mod = ensemble_training(model_RF,model_KNN,model_XGB,X_train,Y_train)

def model_inference(model,X_test,Y_test,model_KNN,model_XGB,model_RF):
    y1 = model_RF.predict(X_test)
    y2 = model_KNN.predict(X_test)
    y3 = model_XGB.predict(X_test)
    X = np.concatenate([y1,y2,y3],axis=1)
    yhat = model.predict(X)
    Y1=scaler2.inverse_transform(yhat[:,0].reshape(-1,1))
    Y2=scaler.inverse_transform(yhat[:,1:].reshape(-1,2))
    yhat = np.concatenate([Y1,Y2],axis=1)
    ytrue = Y_test
    x, y, z = error_calculate(yhat,ytrue)
    return x, y, z

# ## Comparison Between 4 Models

comp_columns = ['Model Name','ROCOF','Settling Frequency','Nadir']
comparison_df = pd.DataFrame(columns = comp_columns)

def find_error(model,X_test,Y_test):
    yhat = model.predict(X_test)
    y1=scaler2.inverse_transform(yhat[:,0].reshape(-1,1))
    y2=scaler.inverse_transform(yhat[:,1:].reshape(-1,2))
    yhat = np.concatenate([y1,y2],axis=1)
    ytrue = Y_test
    x, y, z = error_calculate(yhat,ytrue)
    return x, y, z

model = tf.keras.models.load_model(filepath)
rocof_ANN,setl_ANN,nadir_ANN = find_error(model,X_test,Y_test)
model = model_KNN
rocof_KNN,setl_KNN,nadir_KNN = find_error(model,X_test,Y_test)
model = model_RF
rocof_RF,setl_RF,nadir_RF = find_error(model,X_test,Y_test)
model = model_XGB
rocof_XGB,setl_XGB,nadir_XGB = find_error(model,X_test,Y_test)
model_ensemble = tf.keras.models.load_model(filepath_ens)
rocof_ensemble, setl_ensemble,nadir_ensemble = model_inference(model_ensemble,X_test,Y_test,model_KNN,model_XGB,model_RF)

comparison_df['Model Name'] = ['DNN','KNN','RF','XGB','Merged Model']
comparison_df['ROCOF']= [rocof_ANN,rocof_KNN,rocof_RF,rocof_XGB,rocof_ensemble]
comparison_df['Settling Frequency']= [setl_ANN,setl_KNN,setl_RF,setl_XGB,setl_ensemble]
comparison_df['Nadir']=[nadir_ANN,nadir_KNN,nadir_RF,nadir_XGB,nadir_ensemble]

comparison_df.to_csv("comparison_result_updated.csv")


from django.urls import path, re_path
from .onstart import on_startup
from . import views

app_name = 'receipts'

urlpatterns = [
    # ex: /receipts/pickup/
    path('pickup', views.pickup, name='pickup'),
    # ex: /receipts/pickup/endpoint/
    path('pickup/endpoint', views.pickup_endpoint, name='pickup_endpoint'),
    # ex: /receipts/search
    path('search_receipts', views.search_receipts, name='search_receipts'),
    # ex: /receipts/get_detail/5/
    path('get_detail/<int:receipts_id>', views.get_detail, name='get_detail'),
    # ex: /receipts/get_raw/5
    path('get_raw/<int:receipts_id>', views.get_raw, name='get_raw'),
    # ex: /receipts/sync
    path('sync', views.sync, name='sync'),
    # ex: /receipts/
    re_path(r'^', views.index, name='index'),
]

on_startup()


#!/usr/local/bin/python3

import os, sys, json, time, datetime, math
import pandas as pd
import numpy as np
from scipy.stats import spearmanr, pearsonr
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.pyplot import *
from matplotlib.font_manager import FontProperties
from matplotlib.gridspec import GridSpec
matplotlib.rcParams["pdf.fonttype"] = 42
matplotlib.rcParams["ps.fonttype"] = 42
matplotlib.rcParams["font.size"] = 8
font = FontProperties()
font.set_size(8)
font.set_weight("bold")
font2 = FontProperties()
font2.set_size(7)

# main
if __name__ == "__main__":

    # get path
    sys_path = sys.path[0]
    sep = sys_path.find("src")
    file_path = sys_path[0:sep]
    senti_path = os.path.join(file_path, "data", "corpus_sentiment_w2v_cluster.csv")
    factcheck_path = os.path.join(file_path, "data", "factcheck.csv")
    fig_path = os.path.join(file_path, "results", "factchecked_percentage.pdf")

    # data
    """
    df = pd.read_csv(senti_path)
    factcheck = pd.read_csv(factcheck_path)
    df = df.merge(factcheck, left_on = "id", right_on = "social_id")
    politi = df[df["checkedby"] == "politifact"]
    print(len(politi), len(politi[politi["998"] > 0]))
    snopes = df[df["checkedby"] == "snopes"]
    print(len(snopes), len(snopes[snopes["999"] > 0]))
    """
    politi = [193, 2888 - 193]
    snopes = [107, 1732 - 107]
    explode = [0.2, 0]
    colors = ["seagreen", "indianred"]

    fig, ax = plt.subplots(nrows = 1, ncols = 2, figsize=(3, 1.5))
    _, text = ax[0].pie(politi, startangle = 90, explode = explode, colors = colors, labels = [r"$\bf{politifactref}:$ 6.68%", ""], wedgeprops = {"alpha": 0.9})
    text[0].set_x(1.6)
    text[0].set_y(1.3)
    _, text = ax[1].pie(snopes, startangle = 90, explode = explode, colors = colors, labels = [r"$\bf{snopesref}:$ 6.18%", ""], wedgeprops = {"alpha": 0.9})
    text[0].set_x(1.5)
    text[0].set_y(1.3)
    plt.savefig(fig_path, bbox_inches = "tight", pad_inches = 0)


import cv2

# reading of image
img = cv2.imread("text.jpg")
cv2.imshow("original",img)

# color conversion

img_gray = cv2.cvtColor(img,cv2.COLOR_RGB2GRAY)
img_hsv =  cv2.cvtColor(img,cv2.COLOR_RGB2HSV)

cv2.imshow("grayscale",img_gray)
cv2.imshow("hsv",img_hsv)

# resizing original image

img_resize = cv2.resize(img,(300,300))
cv2.imshow("resized",img_resize)

print("Shape and Size of original image : " , img.shape , img.size)
print("Shape and Size of resized image : " , img_resize.shape , img_resize.size)

# Simple thresholding

ret,thresh_binary = cv2.threshold(img_gray,150,255,cv2.THRESH_BINARY)
ret,thresh_binary_inv = cv2.threshold(img_gray,150,255,cv2.THRESH_BINARY_INV)
ret,thresh_otsu = cv2.threshold(img_gray,150,255,cv2.THRESH_OTSU)

cv2.imshow("simple binary",thresh_binary)
cv2.imshow("simple binary inv",thresh_binary_inv)
cv2.imshow("simple otsu",thresh_otsu)

# adaptive thresholding

ad_thresh_mean = cv2.adaptiveThreshold(img_gray,255,cv2.ADAPTIVE_THRESH_MEAN_C,cv2.THRESH_BINARY,5,2)
ad_thresh_guass = cv2.adaptiveThreshold(img_gray,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY,5,2)

cv2.imshow("adaptive mean",ad_thresh_mean)
cv2.imshow("adaptive guass",ad_thresh_guass)

# saving images

cv2.imwrite("grayscale.jpg",img_gray)
cv2.imwrite("hsv.jpg",img_hsv)
cv2.imwrite("simple_binary.jpg",thresh_binary)
cv2.imwrite("simple_binary_inv.jpg",thresh_binary_inv)
cv2.imwrite("simple_otsu.jpg",thresh_otsu)
cv2.imwrite("adaptive_mean.jpg",ad_thresh_mean)
cv2.imwrite("adaptive_guass.jpg",ad_thresh_guass)

print("ALL IMAGES HAVE BEEN SAVED")

cv2.waitKey(0)
cv2.destroyAllWindows()


# ============================================================================
# FILE: MidiNvim.py
# AUTHOR: Philip Karlsson <philipkarlsson at me.com>
# License: MIT license
# ============================================================================
import neovim
import os
# (only dependency)
import rtmidi
# Need a separate thread for the MIDI
from threading import Thread
import time
from MidiNvim.MidiNvimProgrammer import MidiNvimProgrammer

class MidiNvim(object):
    """ Main class for the plugin """
    def __init__(self, nvim, log=False):
        self.nvim = nvim
        self.midiin = None
        # Shall be parameter
        self.mode = "programmer"
        self.period = 50 # 50ms

# ============================================================================
# Helpers
# ============================================================================
    def thread_midiHandler(self):
        idleCounter = 0
        # This is how we do it
        while self.threadShallRun:
            m = self.midiin.getMessage(self.period)
            idleCounter += self.period
            if m:
                idleCounter = 0
                self.nvim.async_call(self.handleEvent, m, "")
            # Check time intervals every
            elif idleCounter > 250:
                self.nvim.async_call(self.handleEvent, None, "")

    def handleEvent(self, args, range):
        m = args
        self.handler.handleEvent(m)

# ============================================================================
# Commands
# ============================================================================
    def startMidi(self, args, range):
        self.ogBuffer = self.nvim.current.buffer
        # Start the status window
        self.nvim.command('split')
        self.nvim.command('wincmd j')
        self.nvim.command('e midi_status')
        self.nvim.current.window.height = 2
        self.nvim.command('setlocal buftype=nofile')
        self.nvim.command('setlocal filetype=midi_status')
        self.statusBufferNumber = self.nvim.current.buffer.number
        self.statusBuffer = self.nvim.current.buffer
        self.nvim.command('wincmd k')
        self.nvim.current.buffer = self.ogBuffer
        if self.midiin == None:
            self.midiin = rtmidi.RtMidiIn()
        if len(args) > 0:
            port = int(args[0])
        else:
            port = 0
        self.midiin.openPort(port)
        # Shall start a background thread
        self.threadShallRun = True
        self.midiThread = Thread(target=self.thread_midiHandler)
        self.midiThread.start()
        # Setup hander depending on the preset
        if self.mode == "programmer":
            self.handler = MidiNvimProgrammer(self.nvim, self.statusBuffer)
        #
        initialStatStr = "| Port \"" + self.midiin.getPortName(port)
        initialStatStr += "\" opened, waiting for MIDI..."
        self.statusBuffer[0] = initialStatStr
        # Always start in normal mode
        self.statusBuffer.append("| Mode: \"%s\" | Preset: \"%s\"" % ("normal", self.mode))

    def stopMidi(self, args, range):
        # Shall start a background thread
        self.threadShallRun = False

    def listMidiInputs(self, args, range):
        self.midiin = rtmidi.RtMidiIn()
        self.nvim.command('split Midi_Inputs')
        self.nvim.command('setlocal buftype=nofile')
        self.nvim.command('setlocal filetype=midi_inputs')
        portCount = int(self.midiin.getPortCount())
        self.nvim.current.line="~ Listing " + str(portCount) + " MIDI port(s) ~"
        self.nvim.current.buffer.append("--------------------------")
        i = 0
        while(i < portCount):
            self.nvim.current.buffer.append("%d: %s" % (i, self.midiin.getPortName(i)))
            i += 1


from django.shortcuts import render_to_response
from django.template import RequestContext
from models import *
from forms import *

def home(request, *args, **kwargs):
  event = Event.objects.get(pk=1)
  kwargs.update({'event':event})
  context = RequestContext(request, kwargs)
  return render_to_response('homepage/index.html', context)

def form(request, *args, **kwargs):
  event = Event.objects.get(pk=kwargs['event_id'])
  forms = dict()
  inventories = dict()
  for inventory in Inventory.objects.filter(event=event.id):
    item_entries = []
    for item_entry in ItemEntry.objects.filter(inventory=inventory.id):
      item_entry_form = ItemEntryForm(item_entry.get_form_data(), instance=item_entry)
      item_entries.append(item_entry_form)
    inventories[inventory.name] = dict({
                                        'form': InventoryForm(instance=inventory),
                                        'item_entries': item_entries
                                        })
  forms = {
           'event': EventForm(instance=event),
           'inventories': inventories.iteritems(),
           }
  kwargs.update({'forms':forms.iteritems(), 'event': EventForm(instance=event, prefix='event')})
  context = RequestContext(request, kwargs)
  return render_to_response('event/edit_form.html', context)


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

df = pd.read_csv('911.csv')
print(df.info())
print(df.head())

print('Top 5 zipcodes for 911 calls:\n',df['zip'].value_counts().head(5))

print('Top 5 townships for 911 calls:\n',df['twp'].value_counts().head(5))

print('Number of unique title codes: ',df['title'].nunique())

#creating new column 'Reason'
df['Reason'] = df['title'].apply(lambda x: x.split(':')[0])

print('Most common reasons for 911 calls are:\n',df['Reason'].value_counts().head(5))
sns.countplot(df['Reason'])
plt.show()

#converting strings to DateTime objects
df['timeStamp'] = pd.to_datetime(df['timeStamp'])

#creating columns 'Hours', 'Month', 'Day of Week'
df['Hour'] = df['timeStamp'].apply(lambda x: x.hour)
df['Month'] = df['timeStamp'].apply(lambda x: x.month)
df['Day of Week'] = df['timeStamp'].apply(lambda x: x.dayofweek)

dmap = {0:'Mon',1:'Tue',2:'Wed',3:'Thu',4:'Fri',5:'Sat',6:'Sun'}

df['Day of Week'] = df['Day of Week'].map(dmap)
print(df.head())

sns.set()
sns.countplot(x='Day of Week',hue='Reason',data=df)
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
plt.show()

sns.countplot(x='Month',hue='Reason',data=df)
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
plt.show()

byMonth = df.groupby('Month').count()
sns.lmplot(x='Month',y='lat',data=byMonth.reset_index())
plt.show()

#creating column 'Date'
df['Date'] = df['timeStamp'].apply(lambda x: x.date())
print(df.head())

df.groupby('Date').count()['twp'].plot()
plt.tight_layout()
plt.show()

df[df['Reason']=='Traffic'].groupby('Date').count()['twp'].plot()
plt.title('Traffic')
plt.tight_layout()
plt.show()

df[df['Reason']=='Fire'].groupby('Date').count()['twp'].plot()
plt.title('Fire')
plt.tight_layout()
plt.show()

df[df['Reason']=='EMS'].groupby('Date').count()['twp'].plot()
plt.title('EMS')
plt.tight_layout()
plt.show()

heatmapdf = df.groupby(by=['Day of Week','Hour']).count()['Reason'].unstack()
fig = plt.figure(figsize=(12,7))
sns.heatmap(heatmapdf,cmap='coolwarm')
plt.show()
sns.clustermap(heatmapdf,cmap='coolwarm')
plt.show

monthday = df.groupby(by=['Day of Week','Month']).count()['Reason'].unstack()
fig = plt.figure(figsize=(12,7))
sns.heatmap(monthday,cmap='coolwarm')
plt.show()
sns.clustermap(monthday,cmap='coolwarm')
plt.show()


import pandas
import numpy
from utils.bcftools import bcftools_query

def interpolate(vcf_path, cm_path, output_path, chrom):

    genetic_map = pandas.read_csv(cm_path, sep=" ")

    pos = list(map(int, bcftools_query(vcf_path, arg="%POS\n", list_samples=False)))
    # interpolate CM from BP
    #chro = genetic_map[genetic_map.chr == int(chrom)]
    # because for chr19 'Genetic_Map(cM)' is 'Genetic_Map.cM.'
    cms = numpy.interp(pos, genetic_map.position.values, genetic_map.iloc[:, -1].values)

    eps = 1e-7
    new_cms = []
    for i in range(0, cms.shape[0]):
        if i == 0:
            new_cms.append(cms[i])
            continue

        if new_cms[-1] >= cms[i]:
            new_cms.append(new_cms[-1] + eps)
        else:
            new_cms.append(cms[i])

    with open(output_path, 'w') as w_map:
        for i, cm in enumerate(new_cms):
            if i > 0 and cm <= new_cms[i-1]:
                raise ValueError(f'cm positions {i-1} and {i} are NOT strictly increasing: {new_cms[:i+1]}')
            w_map.write(f"{i}\t{cm:.8f}\n")

if __name__ == '__main__':
    vcf_path = snakemake.input['vcf']
    cm_path = snakemake.input['cmmap']

    chrom = snakemake.wildcards['chrom']

    output_path = snakemake.output[0]
    interpolate(vcf_path, cm_path, output_path, chrom)


"""Given an integer k and a string s, find the length of the longest substring
that contains at most k distinct characters.

For example, given s = "abcba" and k = 2, the longest substring with k
distinct characters is "bcb".
"""

from collections import namedtuple

from dcp import harness

def substr(s, k):
    if k < 1 or not s:
        return 0

    max_len = 0
    # keys are the characters in the current substring
    # value is index of the character's last occurrence in the substring
    chars = {}
    # start and end of the current substring
    bounds = 0, 0
    for i, char in enumerate(s):
        chars[char] = i
        if len(chars) <= k:
            new_lowest_bound = bounds[0]
        else:
            new_lowest_bound = chars.pop(min(chars, key=chars.get)) + 1

        bounds = new_lowest_bound, bounds[1] + 1
        max_len = max(max_len, bounds[1] - bounds[0])

    return max_len

if __name__ == "__main__":
    tc = namedtuple("TestCase", "s k expected")
    cases = (
        tc("abcba", 2, 3),
        tc("aabc", 1, 2),
        tc("ceabfddbaa", 3, 5),
        tc("", 3, 0),
    )

    harness(cases, lambda t: substr(t.s, t.k))


import matplotlib.pyplot as plt
import plotly.express as px
import plotly.graph_objects as go
import scipy.io
from plotly.subplots import make_subplots

from globals import *

def load_files():
    data = []
    for mod in signals:
        file_name = pathlib.Path(join(os.getcwd(), 'mat-data', mod + '_features_full_range.mat'))
        file = scipy.io.loadmat(file_name)
        data.append(file[mat_info[mod]])
    return data

def calculate_features_mean(data):
    ft_array = np.array(data)
    ft_mean_array = np.ndarray((len(signals), len(testing_SNR), 1, number_of_used_features))
    for i in range(len(signals)):
        for j in range(len(testing_SNR)):
            for k in range(number_of_used_features):
                ft_mean_array[i, j, 0, k] = np.mean(ft_array[i, j, :, k])
    return ft_mean_array

def calculate_features_stddev(data):
    ft_array = np.array(data)
    ft_mean_array = np.ndarray((len(signals), len(testing_SNR), 1, number_of_used_features))
    for i in range(len(signals)):
        for j in range(len(testing_SNR)):
            for k in range(number_of_used_features):
                ft_mean_array[i, j, 0, k] = np.std(ft_array[i, j, :, k])
    return ft_mean_array

def generate_snr_axis():
    snr_values = np.linspace((testing_SNR[0] - 5) * 2, (testing_SNR[-1] - 5) * 2, len(testing_SNR))
    # Repeat x_axis for all modulations in data
    x_axis = np.ndarray((len(signals), len(testing_SNR)))
    for i in range(len(signals)):
        x_axis[i, :] = snr_values
    return x_axis

def simple_plot(snr_axis, data_axis, plot_type='html', save=True):
    if plot_type == 'html':
        # Plot HTML window using PLOTLY
        fig = make_subplots(rows=5, cols=5, subplot_titles=used_features_names)
        R, C = 1, 1
        for ft in range(number_of_used_features):
            if C == 6:
                R += 1
                C = 1
            for label, signal in enumerate(signals):
                if ft == 0:
                    fig.add_trace(go.Scatter(x=snr_axis[label, :],
                                             y=data_axis[label, :, 0, ft],
                                             legendgroup=signal,
                                             name=signal,
                                             line=dict(color=px.colors.qualitative.Plotly[label])), row=R, col=C)
                else:
                    fig.add_trace(go.Scatter(x=snr_axis[label, :],
                                             y=data_axis[label, :, 0, ft],
                                             legendgroup=signal,
                                             name=signal,
                                             showlegend=False,
                                             line=dict(color=px.colors.qualitative.Plotly[label])), row=R, col=C)

            C += 1
        fig.update_layout(width=1920 * 2, height=1080 * 2, legend=dict(
            orientation="h",
            yanchor="auto",
            y=1.05,
            xanchor="auto",
            x=0,
            title_font_family="Arial",
            title="Modulation",
            font=dict(
                family="Arial",
                size=16,
                color="black"
            )
        ))
        fig.show()
        if save:
            figure_name = pathlib.Path(join(os.getcwd(),
                                            'figures',
                                            'features', '6ft_plots.html'))
            fig.write_html(figure_name.__str__())
            del fig
    elif plot_type == 'png':
        # Plot graphics using only mean (matplotlib.plot)
        for n in range(number_of_used_features):
            plt.figure(num=n, figsize=(6.4, 3.6), dpi=300)
            plt.plot(snr_axis[0, :], data_axis[0, :, 0, n], '#03cffc', linewidth=1.0, antialiased=True)  # BPSK
            plt.plot(snr_axis[1, :], data_axis[1, :, 0, n], '#6203fc', linewidth=1.0, antialiased=True)  # QPSK
            plt.plot(snr_axis[2, :], data_axis[2, :, 0, n], '#be03fc', linewidth=1.0, antialiased=True)  # PSK8
            plt.plot(snr_axis[3, :], data_axis[3, :, 0, n], '#fc0320', linewidth=1.0, antialiased=True)  # QAM16
            plt.plot(snr_axis[4, :], data_axis[4, :, 0, n], 'g', linewidth=1.0, antialiased=True)  # QAM64
            plt.plot(snr_axis[5, :], data_axis[5, :, 0, n], 'k', linewidth=1.0, antialiased=True)  # Noise
            plt.title('Feature ' + str(n + 1) + ' - ' + features_names[n])
            plt.xlabel('SNR')
            plt.ylabel('Value')
            plt.legend(signals)
            figure_name = pathlib.Path(join(os.getcwd(),
                                            'figures',
                                            'features',
                                            'ft_{}_SNR_({})_a_({})_mean.png'.format(str(n + 1),
                                                                                (testing_SNR[0] - 10) * 2,
                                                                                (testing_SNR[-1] - 10) * 2)))
            if save:
                plt.savefig(figure_name, dpi=300, facecolor='w', edgecolor='w',
                            orientation='landscape', format='png',
                            transparent=False, bbox_inches=None, pad_inches=0.1)
                plt.close()
            else:
                plt.show()
            print('Plotting means of feature number {}'.format(n))

def n_frames_plot(n_frames, snr_axis, data_axis, save=False):
    # TODO: HTML plot for all frames
    for n in range(number_of_used_features):
        plt.figure(num=n, figsize=(6.4, 3.6), dpi=300)
        plt.plot(snr_axis[0, :], data_axis[0, :, 0:n_frames, n], '#03cffc', linewidth=1.0, antialiased=True)  # BPSK
        plt.plot(snr_axis[1, :], data_axis[1, :, 0:n_frames, n], '#6203fc', linewidth=1.0, antialiased=True)  # QPSK
        plt.plot(snr_axis[2, :], data_axis[2, :, 0:n_frames, n], '#be03fc', linewidth=1.0, antialiased=True)  # PSK8
        plt.plot(snr_axis[3, :], data_axis[3, :, 0:n_frames, n], '#fc0320', linewidth=1.0, antialiased=True)  # QAM16
        plt.plot(snr_axis[4, :], data_axis[4, :, 0:n_frames, n], 'g', linewidth=1.0, antialiased=True)  # QAM64
        plt.plot(snr_axis[5, :], data_axis[5, :, 0:n_frames, n], 'k', linewidth=1.0, antialiased=True)  # Noise
        plt.xlabel('SNR')
        plt.ylabel('Value')
        plt.title('Feature ' + str(n + 1) + ' - ' + features_names[n])
        # TODO: put modulation names in legend
        # plt.legend(modulation_names)

        figure_name = pathlib.Path(join(os.getcwd(),
                                        'figures',
                                        'features',
                                        'ft_{}_SNR_({})_a_({})_{}_frames.png'.format(str(n + 1),
                                                                                (testing_SNR[0] - 10) * 2,
                                                                                (testing_SNR[-1] - 10) * 2,
                                                                                n_frames)))
        if save:
            plt.savefig(figure_name, dpi=300, facecolor='w', edgecolor='w',
                        orientation='landscape', format='png',
                        transparent=False, bbox_inches=None, pad_inches=0.1)
            plt.close()
        else:
            plt.show()
        print('Plotting 500 frames of feature number {}'.format(n))

def errorbar_plot(snr_axis, mean, stddev, save=False):
    # TODO: HTML plot for errorbar
    # Plot graphics with error bar using standard deviation
    for n in range(number_of_used_features):
        plt.figure(num=n, figsize=(6.4, 3.6), dpi=300)
        plt.errorbar(snr_axis[0, :],
                     mean[0, :, 0, n],
                     yerr=stddev[0, :, 0, n], color='#03cffc')
        plt.errorbar(snr_axis[1, :],
                     mean[1, :, 0, n],
                     yerr=stddev[1, :, 0, n], color='#6203fc')
        plt.errorbar(snr_axis[2, :],
                     mean[2, :, 0, n],
                     yerr=stddev[2, :, 0, n], color='#be03fc')
        plt.errorbar(snr_axis[3, :],
                     mean[3, :, 0, n],
                     yerr=stddev[3, :, 0, n], color='#fc0320')
        plt.errorbar(snr_axis[4, :],
                     mean[4, :, 0, n],
                     yerr=stddev[4, :, 0, n], color='g')
        plt.errorbar(snr_axis[5, :],
                     mean[5, :, 0, n],
                     yerr=stddev[5, :, 0, n], color='k')
        plt.xlabel('SNR')
        plt.ylabel('Value with sigma')
        plt.title('Feature ' + str(n + 1) + ' - ' + features_names[n])
        plt.legend(signals)
        figure_name = pathlib.Path(join(os.getcwd(),
                                        'figures',
                                        'features',
                                        'ft_{}_SNR_({})_a_({})_err.png'.format(str(n + 1),
                                                                               (testing_SNR[0] - 10) * 2,
                                                                               (testing_SNR[-1] - 10) * 2)))
        if save:
            plt.savefig(figure_name, dpi=300, facecolor='w', edgecolor='w',
                        orientation='landscape', format='png',
                        transparent=False, bbox_inches=None, pad_inches=0.1)
            plt.close()
        else:
            plt.show()
        print('Plotting error bar of feature number {}'.format(n))

if __name__ == '__main__':
    # Load files
    files = load_files()
    # Process
    snr_array = generate_snr_axis()
    mean_array = calculate_features_mean(files)
    std_array = calculate_features_stddev(files)
    # Plot
    simple_plot(snr_array, mean_array)
    # n_frames_plot(100, snr_array, np.array(files))
    # errorbar_plot(snr_array, mean_array, std_array)


import json
import sys

def hex2ipv4(ip):
    ipv4 = ""
    count = 0
    while count < 4:
        x = int((ip[count] + ip[count + 1]),16)

        if count == 0:
            ipv4 = str(x)
        else:
            ipv4 = ipv4 + "." + str(x)
        count = count + 1
    return ipv4

with open(str(sys.argv[1])) as f:
    data = json.load(f)

hex_ipv6 = []
hex_ipv4 = []
str_ipv4 = []

for key in data:
    ip = data[key]['ip']
    if ip[25:29] == 'ffff':
        hex_ipv4.append(ip[30:33] + ip[35:])
    else:
        hex_ipv6.append(ip)

for ip in hex_ipv4:
    str_ipv4.append(hex2ipv4(ip))

print(str_ipv4)

f = open("ipv4.txt",'w')
for ip in str_ipv4:
    f.write(ip+",")


import time, json, urllib.request
import luma.core.error
from luma.led_matrix.device import max7219
from luma.core.interface.serial import spi, noop
from luma.core.legacy import show_message
from luma.core.legacy.font import proportional, CP437_FONT

def demo():
    # create matrix device
    serial = spi(port=0, device=0, gpio=noop())
    device = max7219(serial, width=32, height=8, block_orientation=-90, rotate=2)

    # Create The Message
    msg = "Devscover Subscribers: "

    try:
        url = "https://www.googleapis.com/youtube/v3/channels?part=statistics&id=UC7jBsNZf-wRyJkEmDKBrg2A&key=XXXX"
        res = urllib.request.urlopen(url)
        data = json.load(res)
        subs = data['items'][0]['statistics']['subscriberCount']
    except:
        subs='cant load'
    msg_to_show = msg + subs

    # Start The Display
    print(msg_to_show)
    show_message(device, msg_to_show, fill="white", font=proportional(CP437_FONT))
    time.sleep(1)

if __name__ == "__main__":
    try:
        while(True):
            demo()
    except KeyboardInterrupt:
        pass


import os

from scripts.config_parser import get_config
from scripts.get_exif import get_EXIF
from alive_progress import alive_bar

def remove_custom_EXIF(logger, fldr_path):
    try:
        config = get_config()['EXIF']['config']
        command = rf"exiftool -config {config} -all= {os.path.join(fldr_path, 'logs', 'failed_exif')}"
        os.system(command)
        logger.debug(f'removed EXIF data for {fldr_path}')
    except Exception as e:
        logger.error(f"couldn't use custom exif config to remove data: {e}")

def get_tags(logger, fldr_path):
    set_tags = []
    logger.info(os.path.join(fldr_path, 'logs', 'exif', 'exif_out2.txt'))
    try:
        with open(os.path.join(fldr_path, 'logs', 'exif', 'exif_out2.txt')) as f:
            contents = f.read()
        files = contents.split('========')
        tot = len(files)
        with alive_bar(tot, title='Get Tags Progress') as bar:
            for file in files[1:-1]:
                try:
                    data = file.splitlines()
                    if len(data) != 24:
                        for d in data:
                            tags = d.split(':')
                            tag = tags[0]
                            if tag not in set_tags:
                                set_tags.append(tag)
                except Exception as e:
                    logger.warning(f'failed on file {data[0]}: {e}')
                finally:
                    bar()
    except Exception as err:
        logger.critical(f'unable to parse exif: {err}')

    try:
        with open(os.path.join(fldr_path, 'logs', 'exif_to_check.txt'), 'w') as f:
            tot = len(set_tags)
            with alive_bar(tot, title='Write tags Progress') as bar:
                for item in set_tags:
                    if fldr_path not in item:
                        f.write("%s\n" % item)
                    bar()
    except Exception as e:
        logger.critical(f'unable to write tags to file: {e}')

def exif_second_pass(logger, fldr_path):
    remove_custom_EXIF(logger, fldr_path)
    get_EXIF(logger, fldr_path, os.path.join('logs', 'failed_exif'), '2')
    get_tags(logger, fldr_path)


from django.conf.urls import url
from django.contrib import admin
from refactortoscale import views

urlpatterns = [
    url(r'^$', views.home, name="home"),
    url(r'^admin/', admin.site.urls),
    url(r'^basic$', views.submitbasic, name="basic"),
    url(r'^sql$', views.submitsql, name="sql"),
    url(r'^mq$', views.submitmq, name="mq"),
    url(r'^thanks$', views.thanks, name="thanks"),
]