import bpy
from anybase.cls_anyexcept import CAnyExcept
import anyblend

nsh = anyblend.node.shader
nalign = anyblend.node.align

##########################################################
# Create pixel fraction grid group

def Create(*, bForce=False, bUseFakeUser=False):
    """
    Create shader node group for shader label value.
    """

    # Create the name for the sensor specs shade node tree
    sGrpName = "AT.Func.ShaderLabelValue"

    # Try to get ray splitter specification node group
    ngMain = bpy.data.node_groups.get(sGrpName)

    ####!!! Debug
    # bpy.data.node_groups.remove(ngMain)
    # ngMain = None
    ####!!!

    if ngMain is None:
        ngMain = bpy.data.node_groups.new(sGrpName, "ShaderNodeTree")
        bUpdate = True
    else:
        bUpdate = bForce
    # endif

    if bUpdate:
        # Remove all nodes that may be present
        for nod in ngMain.nodes:
            ngMain.nodes.remove(nod)
        # endfor

        ngMain.use_fake_user = bUseFakeUser

        tNodeSpace = (50, 25)
        tNodeSpaceSmall = (25, 15)

        # Define Inputs
        sMaxInstCnt = "Max. Inst. Count"
        sTypeCnt = "Type Count"
        sInstIdx = "Instance Idx"
        sTypeIdx = "Type Idx"

        lInputs = [
            [sMaxInstCnt, "NodeSocketFloat", 1.0],
            [sTypeCnt, "NodeSocketFloat", 1.0],
            [sInstIdx, "NodeSocketFloat", 0.0],
            [sTypeIdx, "NodeSocketFloat", 0.0],
        ]

        # Define Output
        sValueOut = "Shader Label Value"

        lOutputs = [[sValueOut, "NodeSocketFloat"]]

        # Add group inputs if necessary and set default values
        nodIn = nsh.utils.ProvideNodeTreeInputs(ngMain, lInputs)

        skMaxInstCnt = nodIn.outputs[sMaxInstCnt]
        skTypeCnt = nodIn.outputs[sTypeCnt]
        skInstIdx = nodIn.outputs[sInstIdx]
        skTypeIdx = nodIn.outputs[sTypeIdx]

        # Add group outputs if necessary
        nodOut = nsh.utils.ProvideNodeTreeOutputs(ngMain, lOutputs)

        nodIn.location = (-400, 0)

        skTotalCnt = nsh.math.Multiply(ngMain, "Total Count", skMaxInstCnt, skTypeCnt)
        nalign.Relative(nodIn, (1, 0), skTotalCnt, (0, 1), tNodeSpace)

        skInstVal = nsh.math.Multiply(ngMain, "Instance Value", skTypeCnt, skInstIdx)
        nalign.Relative(skTotalCnt, (1, 1), skInstVal, (1, 0), tNodeSpaceSmall)

        skLabelIdx = nsh.math.Add(ngMain, "Label Index", skInstVal, skTypeIdx)
        nalign.Relative(skInstVal, (1, 1), skLabelIdx, (1, 0), tNodeSpaceSmall)

        skMaxVal = nsh.math.Add(ngMain, "Max Value", skTotalCnt, 1.0)
        nalign.Relative(skTotalCnt, (1, 0), skMaxVal, (0, 0), tNodeSpaceSmall)

        skLabelVal = nsh.math.Add(ngMain, "Label Value", skLabelIdx, 1.0)
        nalign.Relative(skLabelIdx, (1, 0), skLabelVal, (0, 0), tNodeSpaceSmall)

        skValue = nsh.math.Divide(ngMain, "Norm. Label Value", skLabelVal, skMaxVal)
        nalign.Relative(skMaxVal, (1, 1), skValue, (0, 0), tNodeSpaceSmall)

        nalign.Relative(skValue, (1, 0), nodOut, (0, 0), tNodeSpace)
        ngMain.links.new(nodOut.inputs[sValueOut], skValue)

        # ###############################################################
    # endif

    return ngMain

# enddef


import os
import numpy as np
from typing import List, Dict, Tuple
import random
from math import *
import gym
import argparse
import time
import numpy as np
import random
import argparse
from collections import namedtuple
import gym
import os
import tqdm

def soft_binarization(x,tau=3):
    return exp(tau*x)/(exp(tau*x)+exp(tau*(1-x)))

initial_holding_allowed=False

class Map():
    def __init__(self,map_size:Tuple[int,int]):
        self.map_size=map_size
        self.map=[['-' for i in range(map_size[0])] for j in range(map_size[1])]
        self.agent=[]
        self.num_agents=2

    def is_boundary(self,i,j):
        return i==0 or j==0 or i==self.map_size[0]-1 or j==self.map_size[1]-1

    def build_boundary(self):
        for i in range(self.map_size[0]):
            for j in range(self.map_size[1]):
                if self.is_boundary(i,j):
                    self.map[i][j]='-'

    # p: percentage of obstacles
    def generate_path(self,p=0.3):
        dist=1/3*(self.map_size[0]+self.map_size[1])
        self.points=[]
        # modify=log(log(self.map_size[0]*self.map_size[1]))
        modify=1
        cnt=round(self.map_size[0]*self.map_size[1]*(1-p)/dist*modify)
        cnt=max(cnt,2)
        for i in range(cnt):
            point=(1+round((self.map_size[0]-3)*soft_binarization(random.random())),1+round((self.map_size[1]-3)*soft_binarization(random.random())))
            self.points.append(point)
            self.map[point[0]][point[1]]='f'
        self.points.append(self.points[0])
        for i in range(cnt):
            cur_x=self.points[i][0]
            cur_y=self.points[i][1]
            while cur_x!=self.points[i+1][0] or cur_y!=self.points[i+1][1]:
                delta_x=np.sign(self.points[i+1][0]-cur_x)
                delta_y=np.sign(self.points[i+1][1]-cur_y)
                if random.random()<0.5:
                    cur_x=cur_x+delta_x
                else:
                    cur_y=cur_y+delta_y
                self.map[cur_x][cur_y]='f'

    def check_connectivity(self,p):
        vis=np.zeros([self.map_size[0],self.map_size[1]],np.int32)
        cc=0
        for i in range(self.map_size[0]):
            for j in range(self.map_size[1]):
                if p[i][j]>0 and vis[i][j]==0:
                    cc+=1
                    dfs_list=[(i,j)]
                    while True:
                        if len(dfs_list)==0:
                            break
                        for ii,jj in dfs_list:
                            if vis[ii][jj]!=0:
                                dfs_list.remove((ii,jj))
                                continue
                            vis[ii][jj]=cc
                            for iii,jjj in [(ii+1,jj),(ii,jj+1),(ii-1,jj),(ii,jj-1)]:
                                if p[iii][jjj]==1 and vis[iii][jjj]==0:
                                    dfs_list.append((iii,jjj))
        for i in range(1,cc+1):
            if np.sum(vis==i)<=2:
                return 0   # CC that consists of less than 2 grids is not allowed
        self.vis=vis
        return cc

    def generate_convex_area(self):
        random_img=np.zeros([self.map_size[0],self.map_size[1]],np.int32)
        num_pts=random.randint(6,10)
        for i in range(num_pts):
            random_img[random.randint(1,self.map_size[0]-2)][random.randint(1,self.map_size[1]-2)]=1

        element_count=np.sum(random_img)
        while True:
            for i in range(self.map_size[0]):
                idx1=np.argwhere(random_img[i]>0)
                if len(idx1)==0:
                    continue
                idx_min=np.min(idx1)
                idx_max=np.max(idx1)
                random_img[i][idx_min:idx_max+1]=1
            for j in range(self.map_size[1]):
                idx1=np.argwhere(random_img[:,j]>0)
                if len(idx1)==0:
                    continue
                idx_min=np.min(idx1)
                idx_max=np.max(idx1)
                random_img[idx_min:idx_max+1][:,j]=1
            if np.sum(random_img)==element_count:
                break
            element_count=np.sum(random_img)

        cc=self.check_connectivity(random_img)
        if cc!=1:
            raise Exception("CC={}".format(cc))

        for i in range(self.map_size[0]):
            for j in range(self.map_size[1]):
                if random_img[i][j]>0:
                    self.map[i][j]='f'

    def generate_two_cc(self):
        while True:
            random_img=np.zeros([self.map_size[0],self.map_size[1]],np.int32)
            for i in range(1,self.map_size[0]-1):
                for j in range(1,self.map_size[1]-1):
                    if random.randint(0,1)==0:
                        random_img[i][j]=1

            cc=self.check_connectivity(random_img)
            if cc==2:
                break
            # else:
            #     print("CC={}".format(cc))

        for i in range(self.map_size[0]):
            for j in range(self.map_size[1]):
                if random_img[i][j]>0:
                    self.map[i][j]='f'

    def generate_fixed_path(self):
        self.map=[['-','-','-','-','-','-','-','-'],
                  ['-','f','f','-','f','f','f','-'],
                  ['-','f','f','-','f','f','f','-'],
                  ['-','f','f','-','f','f','f','-'],
                  ['-','f','f','-','-','f','f','-'],
                  ['-','f','f','f','-','f','f','-'],
                  ['-','f','f','f','-','f','f','-'],
                  ['-','-','-','-','-','-','-','-']]

        self.vis=[[0,0,0,0,0,0,0,0],
                  [0,1,1,0,2,2,2,0],
                  [0,1,1,0,2,2,2,0],
                  [0,1,1,0,2,2,2,0],
                  [0,1,1,0,0,2,2,0],
                  [0,1,1,1,0,2,2,0],
                  [0,1,1,1,0,2,2,0],
                  [0,0,0,0,0,0,0,0]]

    def save(self,path:str="./utils/levels/random/example.txt"):
        with open(path,"w") as f:
            f.write('\n'.join([' '.join(x) for x in self.map]))
            f.write('\n\n')

            for idx,agent in enumerate(self.agent):
                f.write(str(agent[0])+" "+str(agent[1]))
                if initial_holding_allowed:
                    if random.random()<0.2:
                        f.write(" "+random.choice(["o","t","Co","Ct","p","q","pCo","pCt","CoCt","pCoCt","e"]))
                f.write('\n')
    def cnt_grid(self):
        return self.map_size[0]*self.map_size[1]

    def is_occupied(self,i,j):
        return self.map[i][j]!='f' or (j,i) in self.agent

    def count_empty(self):
        cnt=0
        for i in range(self.map_size[0]):
            for j in range(self.map_size[1]):
                if not self.is_occupied(i,j):
                    cnt+=1
        return cnt

    def count_empty_i(self,idx):
        cnt=0
        for i in range(self.map_size[0]):
            for j in range(self.map_size[1]):
                if not self.is_occupied(i,j) and self.vis[i][j]==idx:
                    cnt+=1
        return cnt

    def add_agent(self):
        empty_cnt=self.count_empty()
        x=random.randint(0,empty_cnt-1)
        cnt=0
        for i in range(self.map_size[0]):
            for j in range(self.map_size[1]):
                if not self.is_occupied(i,j):
                    if(cnt==x):
                        self.agent.append((j,i))
                        return
                    cnt+=1

    def add_agent_i(self,idx):
        empty_cnt=self.count_empty_i(idx)
        x=random.randint(0,empty_cnt-1)
        cnt=0
        for i in range(self.map_size[0]):
            for j in range(self.map_size[1]):
                if not self.is_occupied(i,j) and self.vis[i][j]==idx:
                    if(cnt==x):
                        self.agent.append((j,i))
                        return
                    cnt+=1

    def random_accessible_center(self):
        cnt=1
        point=None
        for i in range(self.map_size[0]):
            for j in range(self.map_size[1]):
                if self.map[i][j]=="-":
                    if self.map[max(i-1,0)][j]=='f' or self.map[min(i+1,self.map_size[0]-1)][j]=='f' or self.map[i][max(j-1,0)]=='f' or self.map[i][min(j+1,self.map_size[1]-1)]=='f':
                        if random.random()<1/cnt:
                            point=(i,j)
                        cnt+=1
        if point is None:
            raise ValueError
        return point

    def random_accessible_bridge(self):
        cnt=1
        point=None
        for i in range(self.map_size[0]):
            for j in range(self.map_size[1]):
                if self.map[i][j]=="-":
                    reachable=True
                    for agent_id in range(1,3):
                        if not(self.vis[max(i-1,0)][j]==agent_id or self.vis[min(i+1,self.map_size[0]-1)][j]==agent_id or self.vis[i][max(j-1,0)]==agent_id or self.vis[i][min(j+1,self.map_size[1]-1)]==agent_id):
                            reachable=False
                    if reachable:
                        if random.random()<1/cnt:
                            point=(i,j)
                        cnt+=1
        if point is None:
            raise ValueError
        return point

    def random_accessible_i(self,idx):
        if not isinstance(idx,List):
            idx=[idx]
        cnt=1
        point=None
        for i in range(self.map_size[0]):
            for j in range(self.map_size[1]):
                if self.map[i][j]=="-":
                    reachable=True
                    for agent_id in idx:
                        if not(self.vis[max(i-1,0)][j]==agent_id or self.vis[min(i+1,self.map_size[0]-1)][j]==agent_id or self.vis[i][max(j-1,0)]==agent_id or self.vis[i][min(j+1,self.map_size[1]-1)]==agent_id):
                            reachable=False
                    if not (self.map[max(i-1,0)][j]=='f' or self.map[min(i+1,self.map_size[0]-1)][j]=='f' or self.map[i][max(j-1,0)]=='f' or self.map[i][min(j+1,self.map_size[1]-1)]=='f'):
                        reachable=False
                    if reachable:
                        if random.random()<1/cnt:
                            point=(i,j)
                        cnt+=1
        if point is None:
            raise ValueError
        return point

    def add_delivery(self,idx):
        point=self.random_accessible_i(idx)
        self.map[point[0]][point[1]]='*'

    def add_dishwasher(self,idx):
        point=self.random_accessible_i(idx)
        self.map[point[0]][point[1]]='w'

    def add_fire(self,idx):
        cnt=1
        point=None
        for i in range(self.map_size[0]):
            for j in range(self.map_size[1]):
                if self.map[i][j]=="f" and self.vis[i][j]==idx and not (self.map[max(i-1,0)][j]=='e' or self.map[min(i+1,self.map_size[0]-1)][j]=='e' or self.map[i][max(j-1,0)]=='e' or self.map[i][min(j+1,self.map_size[1]-1)]=='e'):
                    if random.random()<1/cnt:
                        point=(i,j)
                    cnt+=1
        self.map[point[0]][point[1]]='h'

    def add_plate(self,idx):
        point=self.random_accessible_i(idx)
        self.map[point[0]][point[1]]='p'

    def add_random_plate(self,idx):
        point=self.random_accessible_i(idx)
        if random.randint(0,1)==1:
            self.map[point[0]][point[1]]='p'
        else:
            self.map[point[0]][point[1]]='q'

    def add_random_dish(self,idx):
        point=self.random_accessible_i(idx)
        x=random.randint(0,3)
        if x==0:
            self.map[point[0]][point[1]]='pCt'
        elif x==1:
            self.map[point[0]][point[1]]='pCo'
        elif x==2:
            self.map[point[0]][point[1]]='pCoCt'
        elif x==3:
            self.map[point[0]][point[1]]='CoCt'

    def add_random_chopped(self,idx):
        point=self.random_accessible_i(idx)
        if random.randint(0,1)==1:
            self.map[point[0]][point[1]]='Ct'
        else:
            self.map[point[0]][point[1]]='Co'

    def add_random_fresh(self,idx):
        point=self.random_accessible_i(idx)
        if random.randint(0,1)==1:
            self.map[point[0]][point[1]]='t'
        else:
            self.map[point[0]][point[1]]='o'

    def add_object_str(self,s,idx):
        point=self.random_accessible_i(idx)
        self.map[point[0]][point[1]]=s

    def add_vases(self):
        self.map[4][3]='v'
        self.map[4][4]='v'

if __name__=="__main__":
    # training levels for different training stages
    # sast: single agent single subtask, train single agent policies
    # samt: single agent multi subtask, train single agent policies on completing series of subtasks
    # mast: multi agent single subtask
    # mast-cooperation: multi agent single subtask, agent 1 is the leading agent, agent 2 is the follower agent
    # perception: train the perception module
    # mamt: multi agent multi subtask, integrate all the modules
    level=input("level: sast | samt | mast | mast-cooperation | mamt | perception \n")

    os.makedirs(os.path.join("./levels/",level),exist_ok=True)

    if level in ["perception","sast","mast","mast-cooperation"]:
        initial_holding_allowed=True

    program_dir="./programs"

    for i in tqdm.tqdm(range(0,20000)):
        while True:
            try:
                random_cc_agent=random.randint(1,2)
                if level=="sast" or level=="samt":
                    random_cc_fixed=random_cc_agent
                    random_cc_movable=random_cc_agent
                elif level in ["mast","mast","perception","mamt"]:
                    random_cc_fixed=[]
                    random_cc_movable=[]
                elif level=="mast-cooperation":
                    random_cc_fixed=random_cc_agent
                    random_cc_movable=[]
                else:
                    raise ValueError("level not found")
                name="random"+str(i)
                map=Map((8,8))
                map.generate_fixed_path()
                map.add_vases()

                map.add_object_str("*",random_cc_fixed)
                map.add_object_str("w",random_cc_fixed)
                map.add_object_str("T",random_cc_fixed)
                map.add_object_str("O",random_cc_fixed)
                map.add_object_str("/",random_cc_fixed)
                map.add_object_str("e",random_cc_fixed)

                map.add_agent_i(random_cc_agent)
                map.add_agent_i(3-random_cc_agent)
                map.add_agent_i(random_cc_agent)
                map.add_agent_i(3-random_cc_agent)

                # randomly generate one agent and one subtask
                if level in ["sast","mast-cooperation","mast"]:
                    h1=0
                    h2=random.randint(0,15)
                    if h2==0:
                        map.add_object_str("pCoCt",random_cc_movable)
                    if h2==1:
                        map.add_object_str("pCo",random_cc_movable)
                    if h2==2:
                        map.add_object_str("pCt",random_cc_movable)
                    if h2==3:
                        map.add_object_str("p",random_cc_movable)
                        map.add_object_str("CoCt",random_cc_movable)
                    if h2==4:
                        map.add_object_str("p",random_cc_movable)
                        map.add_object_str("Ct",random_cc_movable)
                        map.add_object_str("Co",random_cc_movable)
                    if h2==5:
                        map.add_random_plate(random_cc_movable)
                        map.add_object_str("Co",random_cc_movable)
                        map.add_object_str("Ct",random_cc_movable)
                    if h2==6:
                        map.add_object_str("p",random_cc_movable)
                        map.add_object_str("Co",random_cc_movable)
                        map.add_object_str("Ct",random_cc_movable)
                    if h2==7:
                        map.add_object_str("pCo",random_cc_movable)
                        map.add_object_str("Ct",random_cc_movable)
                    if h2==8:
                        map.add_object_str("pCt",random_cc_movable)
                        map.add_object_str("Co",random_cc_movable)
                    if h2==9 or h2==10:
                        map.add_random_plate(random_cc_movable)
                    if h2==11:
                        map.add_random_plate(random_cc_movable)
                        map.add_object_str("t",random_cc_movable)
                        map.add_object_str("o",random_cc_movable)
                    if h2==12:
                        map.add_random_plate(random_cc_movable)
                        map.add_object_str("t",random_cc_movable)
                        map.add_object_str("o",random_cc_movable)
                    if h2==13 or h2==14:
                        map.add_object_str("q",random_cc_movable)
                        map.add_random_chopped(random_cc_movable)
                    if h2==15:
                        map.add_fire(random_cc_fixed)
                    for _ in range(random.randint(1,3)):
                        map.add_random_dish([])
                    if not level=="sast":
                        if random.random()<0.1 and not h2==15:
                            map.add_fire(random.randint(0,1))

                # randomly generate a perception query
                elif level in ["perception"]:
                    h1=5
                    h2=random.randint(1,4)
                    for _ in range(3):
                        map.add_random_dish([])
                    for _ in range(3):
                        map.add_random_chopped([])
                    if random.random()<0.5:
                        map.add_fire(random.randint(0,1))

                elif level in ["samt","mast-cooperation"]:
                    h1=1
                    h2=random.randint(1,13)
                    if h2==3:
                        map.add_object_str("q",random_cc_movable)
                        map.add_object_str("Ct",random_cc_movable)
                    if h2==4:
                        map.add_object_str("q",random_cc_movable)
                        map.add_object_str("Co",random_cc_movable)
                    if h2==5:
                        map.add_object_str("p",random_cc_movable)
                        map.add_object_str("Co",random_cc_movable)
                    if h2==6:
                        map.add_object_str("p",random_cc_movable)
                        map.add_object_str("Ct",random_cc_movable)
                    if h2==7:
                        map.add_object_str("CoCt",random_cc_movable)
                        map.add_object_str("p",random_cc_movable)
                    if h2==8:
                        map.add_fire(random_cc_movable)
                    if h2==9:
                        map.add_fire(random_cc_movable)
                        map.add_object_str("o",random_cc_movable)
                    if h2==10:
                        map.add_fire(random_cc_movable)
                        map.add_object_str("Ct",random_cc_movable)
                        map.add_object_str("p",random_cc_movable)
                    if h2==11:
                        map.add_fire(random_cc_movable)
                        map.add_object_str("pCoCt",random_cc_movable)
                    if h2==12:
                        map.add_fire(random_cc_movable)
                    if h2==13:
                        map.add_fire(random_cc_movable)
                        map.add_object_str("q",random_cc_movable)

                elif level in ["mamt"]:
                    h1=4
                    h2=random.randint(0,3)
                    map.add_object_str("p",random_cc_movable)
                    map.add_object_str("q",random_cc_movable)
                break
            except:
                continue

        program=open(os.path.join(program_dir,str(h1)+"_"+str(h2)+".txt")).read()
        map.save(os.path.join("./levels/",level,name+".txt"))
        with open(os.path.join("./levels/",level,name+"_program"+".txt"),"w") as f:
            f.write(program)


import os
import xml.sax
import unicodedata

dashes = ['֊', '-', '‐', '‑', '‒', '–', '—', '﹘', '﹣', '－']
correction_regex = r'publisher">[^<]+(Co|Inc|Corp|LP|Crop|corp|Ltd|s\.r\.l|B\.V)</rs>\.'

article_entry = ['TEI']
header_entry = ['teiHeader']
body_entry = ['text']
title_entry = ['title']

def fix_relations(article):
    """Adjust relations to a simple numbering scheme for BRAT annotation

    Args:
        article (dictionary): article information coded in a dictionary
    """
    for rel in article['relations']:
        rel['Arg2'] = article['softcite_id_mapping'][rel['Arg2']]

def repair_citation(text, citation):
    """Adjust article citations to match the "usual" pattern ... [3].

    Args:
        text (string): article text
        citation (string): citation text

    Returns:
        string: adjusted text
    """
    text = text.rsplit(citation, 1)[0]
    add_space = False
    if text.endswith(' '):
        text = text.rstrip()
        add_space = True

    if not text.endswith(' al.') and not text.endswith(' ref.') and text.endswith('.') or text.endswith(',') or text.endswith(';'):
        cut_off = text[-1]
        text = text[:-1]
        text += ' ' + '[' + citation + ']' + cut_off
        if add_space:
            text += ' '
    else:
        if add_space:
            text += ' '
        text += '[' + citation + ']'
    return text

def is_multi_ref(ref):
    """Test if a citation candidate consist of multiple citations

    Args:
        ref (string): citation string

    Returns:
        bool: test result
    """
    comma_count = 0
    dash_count = 0
    digit_count = 0
    for c in ref:
        if c.isdigit():
            digit_count += 1
        elif unicodedata.category(c) == 'Pd':
            dash_count += 1
        elif c == ',':
            comma_count += 1
        else:
            return False
    if ( comma_count > 0 or dash_count > 0 ) and digit_count > 1:
        return True
    else:
        return False

class TEI_Parser(xml.sax.handler.ContentHandler):
    """Parser for TEI XML software annotation.
    """
    def __init__(self):
        # self.entity_type_list = set()
        self.article_count = 0
        self.running_id = 0
        self.running_rel_id = 0
        self.in_article = False
        self.in_header = False
        self.in_body = False
        self.in_title = False
        self.in_id = False
        self.in_origin = False
        self.read_text = False
        self.ref = False
        self.rs = False
        self.current_ref = ''
        self.articles = []

    def add_article(self):
        self.articles.append({
            'text': '',
            'entities': [],
            'relations': [],
            'softcite_id_mapping': {}
        })

    def startElement(self, name, attrs):
        # Recognize when we are dealing with articles or meta-data
        if name in article_entry:
            if attrs['type'] != 'article':
                raise(RuntimeError("Found type {} -- different from articles".format(attrs['type'])))
            self.add_article()
            self.running_id = 0
            self.running_rel_id = 0
            self.article_count += 1
            self.in_article = True
            self.articles[-1]['subtype'] = attrs['subtype']

        # Recognize when we are dealing with the meta-data of a single article
        if self.in_article and name in header_entry:
            self.in_header = True

        if self.in_header:
            if name in title_entry:
                self.in_title = True

        if self.in_header:
            if name == 'idno' and attrs['type'] == 'PMC':
                self.in_id = True

        if self.in_header:
            if name == 'idno' and attrs['type'] == 'origin':
                self.in_origin = True

        # Recognize when we are inside the text of one specific article
        if self.in_article and not self.in_header and name in body_entry:
            self.in_body = True
            if attrs['xml:lang'] != 'en':
                raise(RuntimeError("Non English article in the set."))

        if self.in_body:
            if name not in ['p', 'ref', 'rs', 'text', 'body']:
                raise(RuntimeError("Found unhandled tag: {}".format(name)))
            if name == 'p':
                self.read_text = True
            if name == 'ref' and 'type' in attrs.keys() and attrs['type'] == 'bibr':
                self.ref = True
            if name == 'rs':
                if self.articles[-1]['text'] and not self.articles[-1]['text'].endswith((' ', '(', '[', '{')):
                    self.articles[-1]['text'] += ' '
                self.articles[-1]['entities'].append({
                    'id': self.running_id,
                    'type': attrs['type'],
                    'beg': len(self.articles[-1]['text']),
                    'end': -1,
                    'string': '',
                    'softcite_id': attrs['xml:id'] if 'xml:id' in attrs.keys() else ''
                })
                if 'xml:id' in attrs.keys():
                    self.articles[-1]['softcite_id_mapping'][attrs['xml:id']] = self.running_id
                if 'corresp' in attrs.keys():
                    self.articles[-1]['relations'].append({
                        'id': self.running_rel_id,
                        'type': '{}_of'.format(attrs['type']),
                        'Arg1': self.running_id,
                        'Arg2': attrs['corresp'].lstrip('#')
                    })
                    self.running_rel_id += 1
                # self.entity_type_list.update([attrs['type']])
                self.rs = True
                self.running_id += 1

    def endElement(self, name):
        if name in article_entry:
            self.in_article = False
            if self.articles[-1]['text']:
                if self.articles[-1]['relations']:
                    fix_relations(self.articles[-1])
        if name in header_entry:
            self.in_header = False
        if name in body_entry:
            self.in_body = False
        if name in title_entry:
            self.in_title = False
        if name == 'idno':
            self.in_id = False
            self.in_origin = False
        if name == 'p' and self.read_text:
            self.articles[-1]['text'] += '\n\n'
            self.read_text = False
        if name == 'ref':
            if self.current_ref and ((self.current_ref.isdigit() and len(self.current_ref) < 4 ) or is_multi_ref(self.current_ref) ):
                self.articles[-1]['text'] = repair_citation(self.articles[-1]['text'], self.current_ref)
            self.current_ref = ''
            self.ref = False
        if name == 'rs':
            self.articles[-1]['entities'][-1]['end'] = len(self.articles[-1]['text'])
            self.rs = False

    def characters(self, content):
        if self.in_title:
            self.articles[-1]['title'] = content
        if self.in_id:
            self.articles[-1]['PMC'] = content
        if self.in_origin:
            self.articles[-1]['origin'] = content
        if self.read_text:
            self.articles[-1]['text'] += content
        if self.rs:
            self.articles[-1]['entities'][-1]['string'] = content
        if self.ref:
            self.current_ref += content

def parse(in_file, out_path, write_empty=True):
    """Parse a TEI XML to extract annotate articles and annotation from it.

    Args:
        in_file (PosixPath): file name
        out_path (PosixPath): output path
        write_empty (bool, optional): whether to write empty outputs. Defaults to True.
    """
    parser = xml.sax.make_parser()
    tei_parser = TEI_Parser()
    parser.setContentHandler(tei_parser)

    with in_file.open() as xml_in:
        parser.parse(xml_in)
        print("Parsed {} articles".format(tei_parser.article_count))

    skipped_articles = 0
    for article in tei_parser.articles:
        article_name = article['PMC'] if 'PMC' in article.keys() else article['origin']
        if not write_empty and not article['text'].strip():
            skipped_articles += 1
        else:
            out_text = out_path / '{}.txt'.format(article_name)
            out_annotation = out_path / '{}.ann'.format(article_name)
            with out_text.open(mode='w') as out_art, out_annotation.open(mode='w') as out_anno:
                out_art.write(article['text'])
                for e in article['entities']:
                    out_anno.write('T{}\t{} {} {}\t{}\n'.format(e['id'], e['type'], e['beg'], e['end'], e['string']))
                for r in article['relations']:
                    out_anno.write('R{}\t{} Arg1:T{} Arg2:T{}\t\n'.format(r['id'], r['type'], r['Arg1'], r['Arg2']))
    print("Skipped {} empty articles.".format(skipped_articles))


from bs4 import BeautifulSoup

def my_add(x, y):
    """
    This functions adds x and y args

    Args:
        x: x parameter
        y: y parameter

    Returns:
        x+y

    Examples:
        >>> my_add(1,2)
    """
    return x + y

def parse_dobie_response(xml_response):
    """
    The following function is used to get DOBIE responses and parses annotated tools

    Args:
        xml_response: DOBIE response

    Returns:
        extracted_skills: list of dict
        >>> [{'string': 'CustomerSupport', 'frequencyOfMention': '1', 'kind': 'topic'}, {'string': 'Security', 'frequencyOfMention': '1', 'kind': 'topic'}]

    Examples:
        >>> parse_dobie_response(
        '''<Annotation Id="0" Type="Split" StartNode="42" EndNode="42">
            <Feature>
              <Name className="java.lang.String">kind</Name>
              <Value className="java.lang.String">external</Value>
            </Feature>
        </Annotation>'''
        )
    """
    soup = BeautifulSoup(xml_response, "xml")
    annotations = soup.find_all('Annotation')

    extracted_skills = []

    for annotation in annotations:
        features = annotation.select('Feature')
        skills = dict()

        for feature in features:
            name = feature.Name.text
            value = feature.Value.text

            if value != 'external':
                skills['{}'.format(name)] = value

            if skills:
                extracted_skills.append(skills)
    return extracted_skills


from django.contrib.gis.db.models import PointField
from django.db import models

# from psqlextra.types import PostgresPartitioningMethod
# from psqlextra.models import PostgresPartitionedModel
from model_utils.managers import InheritanceManager

class Vehicle(models.Model):
    custom_partitioned = {
        "column": "id",
        "size": 100000,
    }

    creation_date = models.DateField(
        db_index=True,
    )
    status = models.CharField(
        max_length=20,
        null=True,
        blank=True,
    )
    completion_date = models.DateField(
        null=True,
        blank=True,
        db_index=True,
    )
    service_request_number = models.CharField(
        max_length=50,
        db_index=True,
    )
    type_of_service_request = models.CharField(max_length=255)
    license_plate = models.CharField(
        max_length=1000,
        null=True,
        blank=True,
    )
    vehicle_make = models.CharField(
        max_length=250,
        null=True,
        blank=True,
        db_index=True,
    )
    vehicle_color = models.CharField(
        max_length=250,
        null=True,
        blank=True,
        db_index=True,
    )
    current_activity = models.CharField(
        max_length=250,
        null=True,
        blank=True,
    )
    most_recent_action = models.CharField(
        max_length=250,
        null=True,
        blank=True,
    )
    days_parked = models.BigIntegerField(
        null=True,
        blank=True,
    )
    street_address = models.CharField(
        max_length=250,
        null=True,
        blank=True,
    )
    zip_code = models.CharField(
        max_length=10,
        null=True,
        blank=True,
    )
    x_coordinate = models.FloatField(
        null=True,
        blank=True,
    )
    y_coordinate = models.FloatField(
        null=True,
        blank=True,
    )
    ward = models.IntegerField(
        null=True,
        blank=True,
    )
    police_district = models.IntegerField(
        null=True,
        blank=True,
    )
    community_area = models.IntegerField(
        null=True,
        blank=True,
    )
    ssa = models.IntegerField(
        null=True,
        blank=True,
    )
    latitude = models.FloatField(
        null=True,
        blank=True,
    )
    longitude = models.FloatField(
        null=True,
        blank=True,
    )
    location = PointField(
        null=True,
        blank=True,
    )
    responsible = models.ForeignKey(
        "Responsible",
        null=True,
        on_delete=models.SET_NULL,
    )
    objects = InheritanceManager()

class CriminalVehicle(Vehicle):
    custom_partitioned = {
        "column": "vehicle_ptr_id",
        "size": 100000,
    }
    top_secret = models.BooleanField(default=False)
    police_data = models.JSONField(default=dict)

class MistakeVehicle(Vehicle):
    custom_partitioned = False

    description = models.TextField()

class RepeatedVehicle(Vehicle):
    custom_partitioned = False

    previous_event = models.OneToOneField(
        "Vehicle",
        related_name="next_event",
        on_delete=models.PROTECT,
    )

class Person(models.Model):
    name = models.CharField(max_length=255)
    phone = models.CharField(max_length=50)

    class Meta:
        abstract = True

class Responsible(Person):
    ...

class Reporter(Person):
    reports = models.ManyToManyField(
        "Vehicle",
        related_name="reporters",
    )

class Comment(models.Model):
    custom_partitioned = {
        "column": "vehicle_id",
        "size": 100000,
    }
    vehicle = models.ForeignKey(
        "Vehicle",
        on_delete=models.CASCADE,
    )
    name = models.CharField(max_length=255)
    text = models.TextField()


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

import os

from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from sklearn.preprocessing import StandardScaler, MinMaxScaler

from env import username, password, host

############################# Acquire ###############################

def acquire_df(use_cache=True):
    '''
    This function acquires all necessary housing data from zillow
    needed to better understand future pricing
    '''

    if os.path.exists('zillow.csv') and use_cache:
        print('Using cached csv')
        return pd.read_csv('zillow.csv')
    print('Acquiring data from SQL database')

    database_url_base = f'mysql+pymysql://{username}:{password}@{host}/zillow'
    query = '''
            SELECT prop.*,
               pred.logerror,
               pred.transactiondate,
               air.airconditioningdesc,
               arch.architecturalstyledesc,
               build.buildingclassdesc,
               heat.heatingorsystemdesc,
               landuse.propertylandusedesc,
               story.storydesc,
               construct.typeconstructiondesc
               FROM   properties_2017 prop

               INNER JOIN (SELECT parcelid,
                                  logerror,
                                  Max(transactiondate) transactiondate
                           FROM   predictions_2017
                           GROUP  BY parcelid, logerror) pred
                       USING (parcelid)

               LEFT JOIN airconditioningtype air USING (airconditioningtypeid)
               LEFT JOIN architecturalstyletype arch USING (architecturalstyletypeid)
               LEFT JOIN buildingclasstype build USING (buildingclasstypeid)
               LEFT JOIN heatingorsystemtype heat USING (heatingorsystemtypeid)
               LEFT JOIN propertylandusetype landuse USING (propertylandusetypeid)
               LEFT JOIN storytype story USING (storytypeid)
               LEFT JOIN typeconstructiontype construct USING (typeconstructiontypeid)
               WHERE  prop.latitude IS NOT NULL
               AND prop.longitude IS NOT NULL AND transactiondate <= '2017-12-31'
'''

    df = pd.read_sql(query, database_url_base)
    df.to_csv('zillow.csv', index=False)

    return df

################## Dealing With Missing Values #####################

def handle_missing_values(df, prop_required_column = .5, prop_required_row = .70):
    ''' Takes in a DataFrame and is defaulted to have at least 60% of values for
    columns and 75% for rows'''
    threshold = int(round(prop_required_column * len(df.index),0))
    df.dropna(axis=1, thresh = threshold, inplace = True)
    threshold = int(round(prop_required_row * len(df.columns), 0))
    df.dropna(axis = 0, thresh = threshold, inplace = True)
    return df

def missing_values(df):
    missing_values =pd.concat([
                    df.isna().sum().rename('count'),
                    df.isna().mean().rename('percent')
                    ], axis=1)
    return missing_values

def missing_counts_and_percents(df):
    missing_counts_and_percents = pd.concat([
                                  df.isna().sum(axis=1).rename('num_cols_missing'),
                                  df.isna().mean(axis=1).rename('percent_cols_missing'),
                                  ], axis=1).value_counts().sort_index()
    return pd.DataFrame(missing_counts_and_percents).reset_index()

 ############################ Outliers #############################

def remove_outliers(df, k, cols):
    ''' Take in a dataframe, k value, and specified columns within a dataframe
    and then return the dataframe with outliers removed
    '''
    for col in cols:
        # Get quartiles
        q1, q3 = df[col].quantile([.25, .75])
        # Calculate interquartile range
        iqr = q3 - q1

        upper_bound = q3 + k * iqr   # get upper bound
        lower_bound = q1 - k * iqr   # get lower bound

        # return dataframe without outliers

        df = df[(df[col] > lower_bound) & (df[col] < upper_bound)]

    return df

############################# Clean ################################

def get_counties(df):
    '''
    This function will create dummy variables out of the original fips column.
    And return a dataframe with all of the original columns except regionidcounty.
    We will keep fips column for data validation after making changes.
    New columns added will be 'LA', 'Orange', and 'Ventura' which are boolean
    The fips ids are renamed to be the name of the county each represents.
    '''
    # create dummy vars of fips id
    county_df = pd.get_dummies(df.fips)
    # rename columns by actual county name
    county_df.columns = ['los_angeles', 'orange', 'ventura']
    # concatenate the dataframe with the 3 county columns to the original dataframe
    df_dummies = pd.concat([df, county_df], axis = 1)
    # drop regionidcounty and fips columns
    df_dummies = df_dummies.drop(columns = ['regionidcounty'])
    return df_dummies

def clean_df(df):
    '''
    This function takes in the zillow data, cleans it, and returns a dataframe
    '''
    # Identify the use codes that are single family from SequelAce
    single_fam_use = [261, 262, 263, 264, 265, 266, 268, 273, 275, 276, 279]
    # Make sure the DataFarme only includes the above
    df = df[df.propertylandusetypeid.isin(single_fam_use)]

#     df = get_counties(df)

    # Remove further outliers for sqft to ensure data is usable
    df = df[(df['calculatedfinishedsquarefeet'] > 500)
            & (df['calculatedfinishedsquarefeet'] <3_000)]

    # Remove further outliers for taxvalue to ensure data is usable
    df = df[(df.taxvaluedollarcnt < 3_000_000)]

    # Restrict df to only those properties with at least 1 bath & bed
    df = df[(df.bedroomcnt > 0) & (df.bathroomcnt > 0)]

    # Deal with remaining nulls
    df = handle_missing_values(df, prop_required_column = .5, prop_required_row = .7)

    # Create a column that is the age of the property
    df['age'] = 2022 - df.yearbuilt

    # Create a column for county name based on FIPS
    df['county'] = df.fips.apply(lambda x: 'orange' if x == 6059.0 else 'los_angeles'
                                 if x == 6037.0 else 'ventura')

    # Determine unnecessary columns
    cols_to_remove = ['id','calculatedbathnbr', 'finishedsquarefeet12', 'fullbathcnt',
              'heatingorsystemtypeid', 'heatingorsystemdesc', 'propertycountylandusecode',
              'propertylandusetypeid','propertyzoningdesc', 'regionidcity', 'regionidzip',
              'propertylandusedesc', 'unitcnt', 'censustractandblock', 'buildingqualitytypeid']
     # Create a new dataframe that dropps those columns
    df = df.drop(columns = cols_to_remove)

        #Drop rows with null values since it is only a small portion of the dataframe
    df = df.dropna()

    return df

############################# Split #################################

def split_data(df):
    '''
    This function takes in the dataframe and target variable name as arguments and then
    splits the dataframe into train (56%), validate (24%), & test (20%)
    It will return a list containing the following dataframes: train (for exploration),
    X_train, X_validate, X_test, y_train, y_validate, y_test
    '''
    # split df into train_validate (80%) and test (20%)
    train_validate, test = train_test_split(df, test_size=.20, random_state=1349)
    # split train_validate into train(70% of 80% = 56%) and validate (30% of 80% = 24%)
    train, validate = train_test_split(train_validate, test_size=.3, random_state=1349)

#     #Make copies of train, validate, and test
#     train = train.copy()
#     validate = validate.copy()
#     test = test.copy()

#     # create X_train by dropping the target variable
#     X_train = train.drop(columns=[target_var])
#     # create y_train by keeping only the target variable.
#     y_train = train[[target_var]]

#     # create X_validate by dropping the target variable
#     X_validate = validate.drop(columns=[target_var])
#     # create y_validate by keeping only the target variable.
#     y_validate = validate[[target_var]]

#     # create X_test by dropping the target variable
#     X_test = test.drop(columns=[target_var])
#     # create y_test by keeping only the target variable.
#     y_test = test[[target_var]]

    return train, validate, test,
############################## Scale #################################

def min_max_df(df, features):
    '''
    Scales the df. using the MinMaxScaler()
    takes in the df and returns the df in a scaled fashion.
    '''
    # Make a copy of the original df
    df = df.copy()

    # Create the scaler
    scaler = MinMaxScaler()

    # Fit the scaler
    scaler.fit(df)

    # Transform and rename columns for the df
    df_scaled = pd.DataFrame(scaler.transform(train), columns = train.columns.tolist())
    return df_scaled

def min_max_split(train, validate, test):
    '''
    Scales the 3 data splits. using the MinMaxScaler()
    takes in the train, validate, and test data splits and returns their scaled counterparts.
    If return_scaler is true, the scaler object will be returned as well.
    '''
    # Make the scaler
    scaler = MinMaxScaler()
    # List columns that need to be scaled
    cols = train[['bathroomcnt', 'bedroomcnt',
       'calculatedfinishedsquarefeet',
       'lotsizesquarefeet', 'roomcnt', 'yearbuilt',
       'structuretaxvaluedollarcnt', 'taxvaluedollarcnt',
       'landtaxvaluedollarcnt', 'taxamount', 'taxrate', 'age']].columns.tolist()
    # Make a copy of original train, validate, and test
    train_scaled = train.copy()
    validate_scaled = validate.copy()
    test_scaled = test.copy()
    # Use/fit the Scaler
    train_scaled[cols] = scaler.fit_transform(train[cols])
    validate_scaled[cols] = scaler.fit_transform(validate[cols])
    test_scaled[cols] = scaler.fit_transform(test[cols])

    return train_scaled, validate_scaled, test_scaled

############################# Wrangle ################################

def wrangle_zillow():
    '''
    This function combines both functions above and outputs three
    cleaned and prepped datasets
    '''
    # Acquire the df
    acquire = acquire_df()

    # Get a clean df
    cleaned = clean_df(acquire)

    # Create more features to use
    df = create_features(cleaned)

    # Split that clean df to ensure minimal data leakage
    train, validate, test,
    X_train, X_validate, X_test,
    y_train, y_validate, y_test = split_data(df, 'logerror')

    return train, validate, test, X_train, X_validate, X_test, y_train, y_validate, y_test

############################# Modeling ################################
##################### Show Clusters/Centroids #########################

def cluster(df, feature1, feature2, k):
    X = df[[feature1, feature2]]

    kmeans = KMeans(n_clusters=k).fit(X)

    df['cluster'] = kmeans.labels_
    df.cluster = df.cluster.astype('category')

    df['cluster'] = kmeans.predict(X)

    centroids = pd.DataFrame(kmeans.cluster_centers_, columns=X.columns)

    df.groupby('cluster')[feature1, feature2].mean()

    plt.figure(figsize=(9, 7))

    for cluster, subset in df.groupby('cluster'):
        plt.scatter(subset[feature1], subset[feature2],  label='cluster ' + str(cluster),
                    alpha=.6)

    centroids.plot.scatter(x=feature1, y=feature2, c='black', marker='x', s=100, ax=plt.gca(),
                           label='centroid')

    plt.legend()
    plt.xlabel(feature1)
    plt.ylabel(feature2)
    plt.title('Visualizing Cluster Centers')

    return

################ Find The Best K Value For Clustering ##################

def inertia(df, feature1, feature2, r1, r2):
    cols = [feature1, feature2]
    X = df[cols]

    inertias = {}

    for k in range(r1, r2):
        kmeans = KMeans(n_clusters=k)
        kmeans.fit(X)
        inertias[k] = kmeans.inertia_

    pd.Series(inertias).plot(xlabel='k', ylabel='Inertia', figsize=(9, 7)).plot(marker='x')
    plt.grid()
    return

####################### Plotting Variable Pairs #########################

def plot_variable_pairs(df):
    # plot the columns in a pairplot
    sns.pairplot(df, kind = 'reg', corner = True, plot_kws={'line_kws':{'color':'red'}})
    plt.show()
#     plt.tight_layout()

def create_features(df):
    df['age_bin'] = pd.cut(df.age,
                           bins = [0, 5, 10, 20, 30, 40, 50, 60, 70,
                                   80, 90, 100, 110, 120, 130, 140],
                           labels = [0, .066, .133, .20, .266, .333, .40, .466,
                                     .533, .60, .666, .733, .8, .866, .933])

    # Create taxrate column
    df['taxrate'] = df.taxamount/df.taxvaluedollarcnt*100

    # Create acres column
    df['acres'] = df.lotsizesquarefeet/43560

    # Bin the acres
    df['acres_bin'] = pd.cut(df.acres, bins = [0, .10, .15, .25, .5,
                                               1, 5, 10, 20, 50, 200],
                       labels = [0, .1, .2, .3, .4,
                                 .5, .6, .7, .8, .9])
    # Bin counties
    df['county_code_bin'] = pd.cut(df.fips, bins=[0, 6037.0, 6059.0, 6111.0],
                             labels = ['Los Angeles County', 'Orange County',
                             'Ventura County'])

    # Make an absolute value of logerror column
    df['abs_logerror'] = abs(df.logerror)

    # Bin abs_logerror
    df['abs_logerror_bin'] = pd.cut(df.abs_logerror, [0, .05, .1, .15, .2, .25,
                                                      .3, .35, .4, .45, 5])

    # Bin logerror
    df['logerror_bin'] = pd.cut(df.logerror, [-5, -.2, -.05, .05, .2, 5])

    # Bin sqft
    df['sqft_bin'] = pd.cut(df.calculatedfinishedsquarefeet,
                            bins = [0, 800, 1000, 1250, 1500, 2000,
                                    2500, 3000, 4000, 7000, 12000],
                            labels = [0, .1, .2, .3, .4,
                                      .5, .6, .7, .8, .9]
                       )

    # Dollar/Sqft for structure
    df['structure_dollar_per_sqft'] = df.structuretaxvaluedollarcnt/df.calculatedfinishedsquarefeet

    df['structure_dollar_sqft_bin'] = pd.cut(df.structure_dollar_per_sqft,
                                             bins = [0, 25, 50, 75, 100, 150,
                                                     200, 300, 500, 1000, 1500],
                                             labels = [0, .1, .2, .3, .4,
                                                       .5, .6, .7, .8, .9]
                                            )

    # Dollar/Sqft for land
    df['land_dollar_per_sqft'] = df.landtaxvaluedollarcnt/df.lotsizesquarefeet

    df['lot_dollar_sqft_bin'] = pd.cut(df.land_dollar_per_sqft, bins = [0, 1, 5, 20, 50, 100,
                                                                        250, 500, 1000, 1500, 2000],
                                       labels = [0, .1, .2, .3, .4,
                                                 .5, .6, .7, .8, .9]
                                      )

    # Make bins floats
    df = df.astype({'sqft_bin': 'float64', 'acres_bin': 'float64', 'age_bin': 'float64',
                    'structure_dollar_sqft_bin': 'float64', 'lot_dollar_sqft_bin': 'float64'})

    # Ratio of bathrooms to bedrooms
    df['bath_bed_ratio'] = df.bathroomcnt/df.bedroomcnt

#     # 12447 is the ID for city of LA.
#     # I confirmed through sampling and plotting, as well as looking up a few addresses.
#     df['cola'] = df['regionidcity'].apply(lambda x: 1 if x == 12447.0 else 0)

    return df

def get_zillow_dummies(train, validate, test, cat_columns = ['age_bin', 'county', 'county_code_bin',
                                                             'abs_logerror_bin', 'logerror_bin']):
    '''
    This function takes in train, validate, test and a list of categorical columns for dummies (cat_columns)
    default col_list is for zillow
    '''
    # create dummies
    train = pd.get_dummies(data = train, columns = cat_columns, drop_first=False)
    validate = pd.get_dummies(data = validate, columns = cat_columns, drop_first=False)
    test = pd.get_dummies(data = test, columns = cat_columns, drop_first=False)

    return train, validate, test


##################################################################
# Description: Plot maps of the NH-spatial field of the global sea ice extent at its annual minimum (september) of particular runs of CanESM2-LE when sea ice extent is exceptionnally low and a few years around.
# Code name: plot_sie_september_decrease_map.py
# Date of creation: 2018/04/09
# Date of last modification: 2018/04/09
# Contacts: chavaillaz.yann@ouranos.ca
##################################################################

## Needed packages
import netCDF4 as netcdf                # description of the meaning of data and relations fields stored in a netcdf file
import os                               # portable way of using operating system dependent functionality
import numpy as np                      # scientific computing with Python
import mpl_toolkits                             # extension of matplotlib
from mpl_toolkits.basemap import Basemap
import matplotlib.pyplot as plt
from datetime import datetime as dt
import time as tt
import sys

import functions_plot as fct_pt

# defining variables to plot
nc	= netcdf.Dataset('/exec/yanncha/abrupt_changes/sie_september_decrease_map_CanESM2-LE.nc','r')
lat	= nc.variables['lat'][:]
lon	= nc.variables['lon'][:]
time13	= nc.variables['time13'][:]
sie13	= nc.variables['sie13'][:,:,:]
time50	= nc.variables['time50'][:]
sie50	= nc.variables['sie50'][:,:,:]

# plotting maps
subplt		= [231,232,233,234,235,236]
levels		= [0,10,20,30,40,50,60,70,80,90,100]
palette		= plt.cm.YlGnBu
infinity	= 'neither'

# historical-r2 - r3i1p1
fig1		= plt.figure(figsize=(7.2,5))
for i in range(len(time13)):
	im = fct_pt.map_contourfAR(fig1,subplt[i],sie13[i,:,:],lon,lat,str(time13[i]),"",levels,infinity,palette)
fig1.subplots_adjust(right=0.8)
cbar_ax = fig1.add_axes([0.85, 0.3, 0.01, 0.4])
cbar_ax.set_xlabel('[%]', rotation=0,fontsize=7)
fig1.colorbar(im,cax=cbar_ax)
fig1.suptitle('Sea ice cover [%] in september in the historical-r2 r3i1p1 simulation of CanESM2-LE', fontsize=10)
fig1.show()
fig1.savefig('/exec/yanncha/abrupt_changes/figures/sie_september_decrease_map_r2r3.eps',dpi=300)

# historical-r5 - r10i1p1
fig2		= plt.figure(figsize=(7.2,5))
for i in range(len(time50)):
	im = fct_pt.map_contourfAR(fig2,subplt[i],sie50[i,:,:],lon,lat,str(time50[i]),"",levels,infinity,palette)
cbar_ax = fig2.add_axes([0.85, 0.3, 0.01, 0.4])
fig2.subplots_adjust(right=0.8)
fig2.colorbar(im,cax=cbar_ax)
cbar_ax.set_xlabel('[%]', rotation=0,fontsize=7)
fig2.suptitle('Sea ice cover [%] in september in the historical-r5 r10i1p1 simulation of CanESM2-LE', fontsize=10)
fig2.show()
fig2.savefig('/exec/yanncha/abrupt_changes/figures/sie_september_decrease_map_r5r10.eps',dpi=300)


import copy
import enum
import random
import sys
import termios
import tty

class Direction(enum.Enum):
    UP = 1
    RIGHT = 2
    DOWN = 3
    LEFT = 4

class Game:

    EMPTY_BOARD = [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]

    def __init__(self, score=0, board=EMPTY_BOARD, exp=False):
        self.score = score
        self.board = copy.deepcopy(board)
        self.exp = exp
        if self.board == Game.EMPTY_BOARD:
            self._place_random()
            self._place_random()

    # todo: make this obey `self.exp`
    def get_max_tile(self): return 2**max([max(row) for row in self.board])

    def move(self, direction):
        snapshot = copy.deepcopy(self.board)
        if direction == Direction.RIGHT:
            for r in range(4):
                new, score_change = Game._move_unit(reversed(self.board[r]), self.exp)
                self.board[r] = list(reversed(new))
                self.score += score_change
        elif direction == Direction.LEFT:
            for r in range(4):
                new, score_change = Game._move_unit(self.board[r], self.exp)
                self.board[r] = new
                self.score += score_change
        elif direction == Direction.UP:
            for c in range(4):
                unit = [self.board[0][c], self.board[1][c], self.board[2][c], self.board[3][c]]
                new, score_change = Game._move_unit(unit, self.exp)
                self.board[0][c] = new[0]
                self.board[1][c] = new[1]
                self.board[2][c] = new[2]
                self.board[3][c] = new[3]
                self.score += score_change
        elif direction == Direction.DOWN:
            for c in range(4):
                unit = [self.board[3][c], self.board[2][c], self.board[1][c], self.board[0][c]]
                new, score_change = Game._move_unit(unit, self.exp)
                self.board[0][c] = new[3]
                self.board[1][c] = new[2]
                self.board[2][c] = new[1]
                self.board[3][c] = new[0]
                self.score += score_change

        if self.board != snapshot:
            self._place_random()

    def ongoing(self):
        return self.get_move_mask() != [False] * 4

    def get_move_mask(self):
        moves = set()
        for r, c in [(x, y) for x in range(4) for y in range(3)]:
            if self.board[r][c] == self.board[r][c+1] != 0: moves.update([Direction.LEFT, Direction.RIGHT])
            if self.board[r][c] == 0 and self.board[r][c+1] > 0: moves.add(Direction.LEFT)
            if self.board[r][c] > 0 and self.board[r][c+1] == 0: moves.add(Direction.RIGHT)
        for r, c in [(x, y) for x in range(3) for y in range(4)]:
            if self.board[r][c] == self.board[r+1][c] != 0: moves.update([Direction.UP, Direction.DOWN])
            if self.board[r][c] == 0 and self.board[r+1][c] > 0: moves.add(Direction.UP)
            if self.board[r][c] > 0 and self.board[r+1][c] == 0: moves.add(Direction.DOWN)

        return [
            Direction.UP in moves,
            Direction.RIGHT in moves,
            Direction.DOWN in moves,
            Direction.LEFT in moves
        ]

    def _find_empties(self):
        """Return the index-pairs of all empty board positions"""
        return [(r, c) for r in range(4) for c in range(4) if self.board[r][c] == 0]

    def _place_random(self):
        empties = self._find_empties()
        if empties != []:
            r, c = random.choice(empties)
            self.board[r][c] = 1 if random.random() < 0.9 else 2

    @staticmethod
    def _move_unit(unit, exp=False):
        score_change = 0
        unit = Game._collapse(unit)
        for i in range(3):
            if unit[i] > 0 and unit[i] == unit[i+1]:
                unit[i] += 1
                score_change += 2**unit[i] # todo does this need to be `2**(unit[i]-1)`?
                unit[i+1] = 0
        unit = Game._collapse(unit)
        return unit, score_change

    @staticmethod
    def _collapse(unit):
        unit = [x for x in unit if 0 < x]
        unit += [0] * (4 - len(unit)) # pad the 0s on the right
        return unit

    def _tile_repr(self, tile): return f'{(2**tile if tile > 0 and self.exp else tile):>4}'

    def __repr__(self):
        return "\n".join(["  ".join(self._tile_repr(x) for x in row) for row in self.board])

def play_game():
    def _get_input_char():
        fd = sys.stdin.fileno()
        old_settings = termios.tcgetattr(fd)
        tty.setraw(fd)
        input_char = sys.stdin.read(1)
        termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
        return input_char

    game = Game(exp=True)
    for _ in range(100): print()
    print(game)
    while True:
        move = _get_input_char()
        if move == 'j':
            game.move(Direction.LEFT)
        if move == 'k':
            game.move(Direction.DOWN)
        if move == 'l':
            game.move(Direction.RIGHT)
        if move == 'i':
            game.move(Direction.UP)
        if move == 'q':
            exit()
        for _ in range(100): print()
        print(game)
        print(game.get_move_mask())


# -*- coding: utf-8 -*-
import numpy
import csv
import matplotlib.pyplot as plt
import data_minip.ph_script as calibration
import classes.classes as classes
import matplotlib.font_manager as font_manager
import log.functions
import ui.plot_unbuffered
import glob
import time
import re

from tkinter import messagebox
from matplotlib.backends.backend_pdf import PdfPages
from statsmodels.nonparametric import smoothers_lowess

# ===================================================================================
#  The called procedures
# ===================================================================================
def unbuffered_calculation(o_logbox, _def_values):
    result_dir = _def_values.ub_path
    data_set = []
    uniq_runs = []

    list_files = glob.glob1(result_dir,"*.txt")

    noti_id = []
    # After how many files shall we let the user know the program is still working?
    multiple = 5
    # So long as there are less than 5,000 files the log box will kep the user updated evey 5 processed
    for i in range(1000):
        noti_id.append(i*multiple)
    noti_id.remove(0)

    # Create the pdf for the complete graph output
    pdf_filename = "all_graphs.pdf"
    pdf_filepath = "{0}\\{1}".format(_def_values.ub_path, pdf_filename)
    if _def_values.ub_save_pdf:
        with PdfPages(pdf_filepath) as pdf:
            calculate_file_data(list_files, noti_id, uniq_runs, data_set, _def_values, o_logbox, pdf)
    else:
        calculate_file_data(list_files, noti_id, uniq_runs, data_set, _def_values, o_logbox)

    log.functions.log(True, "info", "All unbuffered data calculated for {0} files!".format(len(list_files)), o_logbox)
    if _def_values.ub_save_pdf:
        log.functions.log(True, "info", r"Saving '{}'".format(pdf_filename), o_logbox)

    # -------------------------------------------------------------------------------
    #  Calculate and save Average pH values
    # -------------------------------------------------------------------------------
    data_set_i = []
    for uniq_id in uniq_runs:
        data_set_i.clear()
        # Get the indexes for a set of repeats
        # INEFICIENT but works
        for i in range(0, len(list_files)):
            if data_set[i].filename_less_repeat == uniq_id:
                data_set_i.append(i)
        _sum_ph = 0
        # Sum the pH values we're interested in
        for i in data_set_i:
            _sum_ph = _sum_ph + data_set[i].calculated_ph

        _average_ph = _sum_ph / len(data_set_i)

        # Save the calculated avg to every item in the data structure
        #  AND save the avg error
        for i in data_set_i:
            data_set[i].set_average_ph = _average_ph
            # Avg pH - filename pH
            data_set[i].avg_error_ph = _average_ph - data_set[i].filename_ph

    # -------------------------------------------------------------------------------
    #  Write the output *.csv
    # -------------------------------------------------------------------------------
    output_file = "{}\\ub_data.csv".format(_def_values.ub_path)
    # First check to see if the file is open!
    try:
        file_o = open(output_file, "w+")
        file_o.close()
    except:
        messagebox.showwarning("Warning!", "The file:\n\'{}\' is open.\nPlease close the file then this message box to continue".format(output_file))

    with open(output_file, "w+") as file:
        csv_file = None
        if file:
            csv_file = csv.writer(file, delimiter=",", quotechar='"', lineterminator='\n')

        if csv_file:
            # -----------------------------------------------------------------------
            #  Write the header
            # -----------------------------------------------------------------------
            headder = []
            headder.append(time.strftime("%c"))
            headder.append("R squared: {}".format(_def_values.cal_rsquared))
            headder.append("Equation: y = {0}x + {1}".format(_def_values.cal_slope, _def_values.cal_intercept))
            csv_file.writerow(headder)

            headder.clear()
            headder = ["Filename",
                       "Unique Group",
                       "Repeat number",
                       "Glass pH",
                       "2nd Derivative",
                       "Calculated pH",
                       "Average pH",
                       "pH error (vs. Average)",
                       "pH error (vs. Run)"]
            csv_file.writerow(headder)

            # -----------------------------------------------------------------------
            #  Write the meaty part!
            # -----------------------------------------------------------------------
            for i in range(len(data_set)):
                # None if the filename was skipped due to being poorly named
                if data_set[i].filename_ph != None:
                    _filename_ph = "{:.2f}".format(data_set[i].filename_ph)
                else:
                    _filename_ph = ""
                csv_file.writerow([data_set[i].filename,
                                   data_set[i].filename_less_repeat,
                                   data_set[i].repeat_number,
                                   _filename_ph,
                                   data_set[i].second_der,
                                   data_set[i].calculated_ph,
                                   data_set[i].set_average_ph,
                                   data_set[i].avg_error_ph,
                                   data_set[i].error_ph])
            log.functions.log(True, "info", r"Finished saving 'ub_data.csv'", o_logbox)

    # -------------------------------------------------------------------------------
    #  Find unique repeat numbers
    # -------------------------------------------------------------------------------
    list_rn = []
    # Create a list if unique repeat numbers (removing any characters not in the
    #  range 0-9)
    for i in range(0, len(data_set)):
        if data_set[i].repeat_number != None:
            cur_rn = re.sub(r"\D", "", str(data_set[i].repeat_number))
            if cur_rn not in list_rn:
                log.functions.log(False, "debug", r"unique run number '{}' discovered! saving".format(cur_rn), o_logbox)
                list_rn.append(cur_rn)

    # Make sure the checkboxes will be in order in the toplevel form
    list_rn.sort()
    list_rn.insert(0, "Average")

    ui.plot_unbuffered.show(_def_values, o_logbox, list_rn, data_set)

# Called by the ui for plotting unbuffered data
def plot(checkbox_list, def_values, o_logbox, data_set, _toplevel):
    _legend = []
    fig, ax = plt.subplots(figsize=(6, 5))
    for i in range(0, len(checkbox_list)):
        # If we wanted to plot this value
        if checkbox_list[i][1].get() == True:
            if checkbox_list[i][0] == "Average":
                _legend.append("Average Error")
            else:
                _legend.append("Run {} Error".format(checkbox_list[i][0]))

            x_vals = []
            y_vals = []
            done_avg = []
            for j in range(0, len(data_set)):
                if data_set[j].repeat_number != None:
                    # If our repeat number is in the data_set repeat number plot the value
                    if str( checkbox_list[i][0]) in str(data_set[j].repeat_number):
                        x_vals.append(data_set[j].filename_ph)
                        y_vals.append(data_set[j].error_ph)
                    elif checkbox_list[i][0] ==  "Average":
                        # So we don't plot each average n repeat times
                        if data_set[j].filename_less_repeat not in done_avg:
                            x_vals.append(data_set[j].filename_ph)
                            y_vals.append(data_set[j].avg_error_ph)
                            done_avg.append(data_set[j].filename_less_repeat)

            plt.scatter(x_vals, y_vals)

    # Remove the user selection pane
    _toplevel.destroy()

    # Modify the graph elements
    _fontsize = 8
    draw_graph(ax, "pH error in unbuffered", "Measured pH (Glass Probe)", "pH error", _legend=True, _legend_data=_legend, _sci=False, _fontsize=_fontsize, _bold=True, _hline=True)
    _figure = "{}\\unbuffered_scatter.png".format(def_values.ub_path)
    log.functions.log(True, "info", "Saving figure: unbuffered_scatter.png", o_logbox)
    fig.savefig(_figure, bbox_inches = 'tight')
    plt.show(fig)
    plt.close(fig)

# ===================================================================================
#  Internal procedures
# ===================================================================================
def calculate_file_data(_list_files, _noti_id, _uniq_runs, _data_set, _def_values, _o_logbox, _pdf=None):
    for i in range(0, len(_list_files)):
        if i in _noti_id:
            log.functions.log(True, "info", r"Files processed:{:4d}".format(i), _o_logbox)
        # Let's assume the filename is named correctly
        good_filename = True

        split_filename = _list_files[i].split("_")
        split_filename_len = len(split_filename)

		# This is because the defined filename structure requires at least 2
		#  underscores, and hence 3 sections. [identifier]_[pH]_[repeat-number]
        if split_filename_len <= 2:
            good_filename = False
        # Even if the filename is bad, we'll make a class that won't be used
        #  Otherwise the index would be one out in the for loop
        _data_set.append(classes.UnbufferedData())
        _data_set[i].filename = _list_files[i]

        # ---------------------------------------------------------------------------
        #  Set reference filename & filename_less_repeat
        # ---------------------------------------------------------------------------
        _data_set[i].fullpath = "{0}\\{1}".format(_def_values.ub_path, _data_set[i].filename)
        log.functions.log(False, "debug", r"Setting reference filename '{0}'".format(_data_set[i].filename), _o_logbox)

        if good_filename:
            # -----------------------------------------------------------------------
            #  Calculate and set the 2nd derivative
            # -----------------------------------------------------------------------
            # Calculate the 2nd derivative and check the validity of the data
            #  The validity check purely checks for if there is a positive current value
            #   in the graph, if there is the user is questioned
            returned = process_data_file_ub(_data_set[i].fullpath, _o_logbox, _def_values, _pdf)
            crossing = returned[0]
            good_data = returned[1]
            if good_data:
                log.functions.log(False, "debug", r"Calculation complete! for '{0}' | Value: {1}".format(_data_set[i].filename, crossing), _o_logbox)
                # Save the second derivative to the data_set
                _data_set[i].second_der = crossing

                # Create a list of unique runs
                del(split_filename[-1])
                _data_set[i].filename_less_repeat = "_".join(split_filename)
                if _data_set[i].filename_less_repeat not in _uniq_runs:
                    _uniq_runs.append(_data_set[i].filename_less_repeat)

                # -------------------------------------------------------------------
                #  Set repeat number - 0 if null value
                # -------------------------------------------------------------------
                split_filename = _data_set[i].filename.split("_")
                # List index of final section "_[repeat_number].txt"
                list_index = len(split_filename) - 1
                if split_filename[list_index] != ".txt":
                    split_filename_dot = split_filename[list_index].split(".")
                    _data_set[i].repeat_number = split_filename_dot[0]
                    log.functions.log(False, "debug", r"Repeat number for filename '{0}' set as '{1}'".format(_data_set[i].filename, _data_set[i].repeat_number), _o_logbox)
                else: # Edge case for pHX,XX_.txt with no repeat number selected
                    _data_set[i].repeat_number = 0
                    log.functions.log(False, "debug", r"Repeat number for filename '{0}' set as '0'".format(_data_set[i].filename), _o_logbox)

                # -------------------------------------------------------------------
                #  Set the filename pH
                # -------------------------------------------------------------------
                # List index of penultimate section "_[repeat_number].txt"
                list_index = len(split_filename) - 2
                # Replace the comma with a decimal point
                _ph = split_filename[list_index].replace(",", ".")
                # Remove the "pH" from the string
                _ph = _ph[2:]
                # Cast _ph to a float
                _data_set[i].filename_ph = float(_ph)
                log.functions.log(False, "debug", r"Filename pH for filename '{0}' set as '{1}'".format(_data_set[i].filename, _data_set[i].filename_ph), _o_logbox)

                # -------------------------------------------------------------------
                #  Calculate pH & error
                # -------------------------------------------------------------------
                y = float(_data_set[i].second_der)
                c = float(_def_values.cal_intercept)
                m = float(_def_values.cal_slope)
                _data_set[i].calculated_ph = float((y - c) / m)

                # Error (calculated pH - file pH)
                _data_set[i].error_ph = _data_set[i].calculated_ph - _data_set[i].filename_ph
            else:
                # Bad Data
                log.functions.log(True, "info", r"The file '{0}' is not valid data Skipping!".format(_list_files[i]), _o_logbox)
                _data_set[i].filename_less_repeat = "Skipped! Invalid data"
        else:
            # Bad filename
            _data_set[i].filename_less_repeat = "Skipped! Incorrect file name formating"
            log.functions.log(True, "info", r"The file '{0}' is not named correctly! Skipping!".format(_list_files[i]), _o_logbox)

def find_sequential_sublists(_y, _list_of_sublists, _def_values):
    neg_i = []
    pos_i = []
    # Create 2 lists: one with the indexes of the +ve numbers, the other with -ve
    #  numbers for the 2nd derivative
    for i, val in  enumerate(_y):
        if val >= 0:
            pos_i.append(i)
        else:
            neg_i.append(i)

    temp_list = []
    prev_value = pos_i[0]
    temp_list.append(pos_i[0])
    for i in range(1, len(pos_i)):
        if pos_i[i] == prev_value + 1:
            temp_list.append(pos_i[i])
            prev_value = pos_i[i]
        else:
            # 3 membered tuple used for later use: [sublist, length, max 2nd der value, index of max 2nd der value]
            _list_of_sublists.append([temp_list, len(temp_list), None, None])
            temp_list = []
            temp_list.append(pos_i[i])
            prev_value = pos_i[i]

    _list_of_sublists.append([temp_list, len(temp_list), None, None])

    # -------------------------------------------------------------------------------
    #  Remove any sublists that start with the zeroth index OR end at the final index
    # -------------------------------------------------------------------------------
    # Rather than remove the dataset entirely let's chop of the last x data points so
    #  any sharp spikes don't ruin the data set. UNLESS the list is < x long, in
    #  which case we remove the sublist
    #
    # x is calculated by dividing the length of the data set by the truncation
    #  factor. The size of the dataset (with a truncation factor of 10) is ~100 for
    #  1100 data points, and ~50 for 550
    x = int(len(_y) / _def_values.ub_truncate_factor)
    if _list_of_sublists[0][0][0] == 0:
        # If the length of the sublist is MORE than the number of items we're
        #  removing, then you can remove x items.
        if len(_list_of_sublists[0][0]) > x:
            _list_of_sublists[0][0] = _list_of_sublists[0][0][x:]
        # Else remove the list, as it is less than x items long. This can be done
        #  because if the list is < x long it is not relevant
        else:
            del _list_of_sublists[0]

    last_list = len(_list_of_sublists) - 1
    last_index = len(_list_of_sublists[last_list][0]) - 1
    final_index = len(_y) - 1
    if _list_of_sublists[last_list][0][last_index] == final_index:
        # If the length of the sublist is MORE than the number of items we're
        #  removing, then you can remove x items.
        if len(_list_of_sublists[last_list][0]) > x:
            _list_of_sublists[last_list][0] = _list_of_sublists[last_list][0][:-x]
        # Else remove the list, as it is less than x items long. This can be done
        #  because if the list is < x long it is not relevant
        else:
            del _list_of_sublists[last_list]

def find_max_sec_der(_y, _list_of_sublists):
    j = 0
    for sublist, length, max_value, max_index in _list_of_sublists:
        max_value = _y[sublist[0]]
        for i in range(1, len(sublist)):
            if _y[sublist[i]] > max_value:
                max_value = _y[sublist[i]]
                max_index = sublist[i]
        _list_of_sublists[j][2] = max_value
        _list_of_sublists[j][3] = max_index
        j += 1

# Returns the index of the sublist which contains the max value
def pick_sublist(_list_of_sublists):
    temp_list = []
    # Final index 1 = max sublist size
    # Final index 2 = max 2nd der
    #  This is based on the 2 membered tuple created previously
    final_i = 2
    for i in range(len(_list_of_sublists)):
        temp_list.append(_list_of_sublists[i][final_i])

    return temp_list.index(max(temp_list))

def draw_graph(_ax, _title, _xlabel, _ylabel, _fontsize=8, _hline=False, _draw=False, _x=None, _y=None, _linewidth=1, _sci=True, _grid=True, _bold=False, _legend=False, _legend_data=None):
    if _draw:
        _ax.plot(_x, _y, linewidth = _linewidth)
    if _bold:
        _ax.set_title(_title, fontsize = _fontsize + 2, fontweight = "bold")
    else:
        _ax.set_title(_title, fontsize = _fontsize)
    _ax.set_xlabel(_xlabel, fontsize = _fontsize)
    _ax.set_ylabel(_ylabel, fontsize = _fontsize)
    if _legend:
        _ax.legend(_legend_data)
    if _grid:
        _ax.set_axisbelow(True)
        _ax.tick_params(axis = "both", direction = "out", labelsize = _fontsize)
        _ax.yaxis.grid(color = "lightgray", linestyle = "-", linewidth = _linewidth)
        _ax.xaxis.grid(color = "lightgray", linestyle = "-", linewidth = _linewidth)
    if _hline:
        _ax.axhline(0, color = "black", linewidth = _linewidth)
    if _sci:
        _ax.ticklabel_format(style = 'sci', axis = 'y', scilimits = (0,0), fontsize = _fontsize)
        _ax.yaxis.offsetText.set_fontsize(_fontsize)

# Draws the elements of one of the six subfigures contained within each page of the PDF file
def plot_subfig(x, y, _title, axis, row, column, fullpath, _xlabel, _ylabel, _fontsize=8, hline=False, draw=True):
    if draw:
        if hline:
            draw_graph(axis[row, column], _title, _xlabel, _ylabel, _draw=True, _x=x, _y=y, _hline=True)
        else:
            draw_graph(axis[row, column], _title, _xlabel, _ylabel, _draw=True, _x=x, _y=y)
    else:
        with open(fullpath) as file:
            lines = file.read().splitlines()
        date = "{:<25}\n{:>29}".format("Date of data collection:", lines[0])
        for line in lines:
            if "Instrument Model:" in line:
                line = line.rpartition(":")[2].strip()
                instrument = "{:<25}{}".format("Instrument Model:", line)
            elif "Incr E (V)" in line:
                line = line.rpartition("=")[2].strip()
                incr_e = "{:<25}{}".format("Increment E (V):", line)
            elif "Amplitude (V)" in line:
                line = line.rpartition("=")[2].strip()
                amplitude = "{:<25}{}".format("Amplitude (V):", line)
            elif "Frequency (Hz)" in line:
                line = line.rpartition("=")[2].strip()
                frequency = "{:<25}{}".format("Frequency (Hz):", line)
            elif "Sensitivity (A/V)" in line:
                line = line.rpartition("=")[2].strip()
                sensitivity = "{:<25}{}".format("Sensitivity (A/V):", line)

        _data = "File Metadata\n\n{}\n{}\n{}\n{}\n{}\n{}".format( date,
                                                                  instrument,
                                                                  incr_e,
                                                                  amplitude,
                                                                  frequency,
                                                                  sensitivity)

        o_font = font_manager.FontProperties()
        o_font.set_family(o_font)
        o_font.set_family("monospace")
        axis[row, column].text( 0,
                                1,
                                _data,
                                size=_fontsize + -2,
                                ha='left',
                                va='top',
                                fontproperties=o_font)
        axis[row, column].axis("off")

# Modified from ph_script
def process_data_file_ub(datafile, o_logbox, def_values, pdf):
    # Let's assume we have a valid dataset
    _good_data = True
    # For log message formatting
    _filename = datafile.rpartition("\\")[2]

    start_row = calibration.find_file_start_row(datafile)
    _delimiter = calibration.calculate_delimiter(datafile, start_row)
    data = numpy.genfromtxt(datafile, skip_header=start_row + 1, delimiter=_delimiter)

    # Calculate the moving average (window 20)
    x = data[:,0]
    y = data[:,1]

    # Is it a valid data set? (all values < 0)
    if y.max() > 0:
        messagebox.showwarning("Warning!", "The file:\'{}\' has been flagged as an invalid dataset. After you close this message box the data in question will be shown to you. Please determine if it is indeed invalid data, then close the graph.\nA further dialog will be provided after the graph is closed.".format(_filename))

        fig, ax = plt.subplots(figsize = (6, 5))
        _fontsize = 8
        draw_graph(ax, "Potential Erroneous Data", "Voltage / V", "Current / A", _grid=True, _sci=True, _draw=True, _x=x, _y=y, _bold=True, _fontsize=_fontsize)
        plt.show(fig)
        plt.close()

        if messagebox.askyesno("Further Information Required!", "Was the data invalid?"):
            _good_data = False
    _error = False
    if _good_data:
        y_avg = calibration.movingaverage(y, 20)

        log.functions.log(False, "debug", r"Calculating first deferential for '{0}'".format(_filename), o_logbox)
        # Calculate the first differential
        diff_y_avg = calibration.differentiate(x, y_avg)

        log.functions.log(False, "debug", r"Calculating second deferential for {0}".format(_filename), o_logbox)
        # Calculate the second differential
        diff2_y_avg = calibration.differentiate(x, diff_y_avg)
        # Lowess smooth the second differential
        low = smoothers_lowess.lowess(diff2_y_avg, x, frac=0.1)
        # lowess helpfully sorts out x parameters, we want them reversed
        low_x = numpy.flip(low[:,0], 0)
        low_y = numpy.flip(low[:,1], 0)

        # ---------------------------------------------------------------------------
        #  Calculate the large moving average and 2nd derivative (250 point moving
        #   average is the default - value changed in the default.ini file)
        # ---------------------------------------------------------------------------
        # _points is the number of points used in the very smoothed moving average
        #  The default value for the moving average factor is 10 which gives a moving
        #  average of 110 points with a data set size 1100, and a 55 point moving
		#  average width with a 550 size data set
        _points = int(len(y) / def_values.ub_ma_point_factor)
        # N.B: The "vs_" prefix = very smoothed
        vs_y = calibration.movingaverage(y_avg, _points)
        vs_diff_y_avg = calibration.differentiate(x, vs_y)
        vs_diff2_y_avg = calibration.differentiate(x, vs_diff_y_avg)
        # Lowess smooth the second differential
        vs_low = smoothers_lowess.lowess(vs_diff2_y_avg, x, frac=0.1)
        # lowess helpfully sorts out x parameters, we want them reversed
        vs_low_y = numpy.flip(vs_low[:,1], 0)

        # ---------------------------------------------------------------------------
        #  Saving the pdf
        # ---------------------------------------------------------------------------
        if def_values.ub_save_pdf:
            # Debuging graphs used to understand the shape of the data, and ergo how
            #  to algorithmically pick the correct 2nd derivative
            _fontsize = 8
            # 8.27, 11.69 inces is A4
            fig, axis = plt.subplots(nrows = 3, ncols = 2, figsize = (8.27, 11.69))
            # Draw subgraphs
            x_label = "Voltage / V"
            y_label = "Current / A"
            # Raw data & smoothed data
            plot_subfig([], [], "", axis, 0, 0, datafile, x_label, y_label, _fontsize=_fontsize, draw = False)
            plot_subfig(x, y_avg, "Plot of 20 point moving\naverage input data", axis, 1, 0, datafile, x_label, y_label, _fontsize=_fontsize)
            # \n\n for padding from the bold title at the top of the page
            plot_subfig(x, y, "\n\nPlot of input data", axis, 0, 1, datafile,x_label, y_label, _fontsize=_fontsize)
            plot_subfig(x, vs_y, "Plot of smoothed {}\npoint moving average".format(_points), axis, 1, 1, datafile, x_label, y_label, _fontsize=_fontsize)
            # 2nd derivative data
            y_label = "f''(Current) / AV⁻²"
            plot_subfig(low_x, low_y, "2nd derivative of the 20 point\n moving average, then lowess smoothed", axis, 2, 0, datafile, x_label, y_label, _fontsize=_fontsize, hline = True)
            plot_subfig(low_x, vs_low_y, "2nd derivative of the {} point\n moving average, then lowess smoothed".format(_points), axis, 2, 1, datafile, x_label, y_label, _fontsize=_fontsize, hline = True)
            fig.suptitle(r"Graphs for file: '{}'".format(_filename), fontsize = _fontsize, fontweight = "bold")
            fig.tight_layout()
            pdf.savefig(fig)
            # Uncomment to show each page of the PDF as it is being saved
            #plt.show(fig)
            plt.close(fig)

        crossing_x, _error = find_zero_crossing_point_mod(low_x, low_y, vs_low_y, o_logbox, def_values)
    else:
        crossing_x = None

    if _error:
        _good_data = False
        log.functions.log(True, "warning", r"The issue was with the file '{}'".format(_filename), o_logbox)
    return [crossing_x, _good_data]

# Redesigned from ph_script - 3 lines remain in tact!
def find_zero_crossing_point_mod(x, y, vs_y, o_logbox, _def_values):
    """
    We want to find x coord where our signal crosses the zero point
    However we want to ignore any earlier fluctuations, so we first find
    the lowest point and trace back from there until we find 0.

    Logic redesigned by Harry Tunstall and documented in the supporting
    document (./aion-master/docs).
    """
    # -------------------------------------------------------------------------------
    #  Start of new logic
    # -------------------------------------------------------------------------------
    #  This logic is designed to discover where the true turning point is. It is
    #   fully documented in the supporting document. Helpful comment lines will be
    #   placed to aid understanding.
    #
    # ===============================================================================
    #  Normal smoothed data analysis
    # ===============================================================================
    # Find and save all sequential sublists
    list_of_sublists = []
    find_sequential_sublists(y, list_of_sublists, _def_values)
    # Find max 2nd derivative value of each sublist
    find_max_sec_der(y, list_of_sublists)

    # ===============================================================================
    #  Very smoothed data analysis
    # ===============================================================================
    # Find and save all sequential sublists
    vs_list_of_sublists = []
    find_sequential_sublists(vs_y, vs_list_of_sublists, _def_values)
    # Find max 2nd derivative value of each sublist
    find_max_sec_der(vs_y, vs_list_of_sublists)

    # -------------------------------------------------------------------------------
    #  Which 2nd derivative sublist will you pick!
    # -------------------------------------------------------------------------------
    vs_max_i = pick_sublist(vs_list_of_sublists)
    # vs_list_of_sublists[vs_max_i][3] = index of the largest 2nd derivative value
    vs_mid_i = vs_list_of_sublists[vs_max_i][3]

    # Find which sublist in the normal data is the correct one
    max_i = None
    for i in range(len(list_of_sublists)):
        # If the index is contained within the sublist of indexes
        if vs_mid_i in list_of_sublists[i][0]:
            max_i = i

    # Since i is used in the original code to denote the first +ve index after the
    #  turning point we must set it accordingly
    _error = False
    if max_i != None:
        i = list_of_sublists[max_i][0][0]
    else:
        _error = True
        log.functions.log(True, "warning", r"The turning point could not be found!", o_logbox)
        i = list_of_sublists[max_i][0][0]
    # -------------------------------------------------------------------------------
    #   End of new logic
    # -------------------------------------------------------------------------------
    # i is the first value greater than zero after our turning point. We get the x, y
    #  coords of the points directly before and after crossing zero.
    x = x[i-1:i+1]
    y = y[i-1:i+1]

    # We want to project where we actually would've crossed zero and plot that
    if _error == False:
        crossing = numpy.interp(0, y, x)
    else:
        crossing = None
    return [crossing, _error]


import os
import json

from requests_oauthlib import OAuth1Session #OAuthのライブラリの読み込み

current_path = os.getcwd()
twitter_config_path = os.path.join(current_path, 'notify/twitter')
if "twitter_api_config.py" in os.listdir(twitter_config_path):
    import notify.twitter.twitter_api_config as config

def tweetInfo(image_files=None):
    """
    引数に画像のファイル名のリストを渡し、画像をツイートする

    Args:
        image_files     --- 画像の保存ファイル名のリスト
    Return:
        twitter_url     --- ツイートのURL

    """
    CK = config.CONSUMER_KEY
    CS = config.CONSUMER_SECRET
    AT = config.ACCESS_TOKEN
    ATS = config.ACCESS_TOKEN_SECRET
    twitter = OAuth1Session(CK, CS, AT, ATS) #認証処理

    if (image_files is not None) or (len(image_files) < 5 and len(image_files) > 0):
        url = "https://api.twitter.com/1.1/statuses/update.json"
        url_media = "https://upload.twitter.com/1.1/media/upload.json"

        media_ids = ""
        for image_filepath in image_files:
            ## ツイート画像 ##
            files = {"media" : open(image_filepath, 'rb')}
            req_media = twitter.post(url_media, files=files)

            if req_media.status_code != 200:
                print ("画像アップデート失敗: {}".format(req_media.text))
                exit()

            # Media ID を取得
            media_id = json.loads(req_media.text)['media_id']
            media_id_string = json.loads(req_media.text)['media_id_string']

            print ("Media ID: %d" % media_id)

            if media_ids == "":
                media_ids += media_id_string
            else:
                media_ids = media_ids + "," + media_id_string

        ## ツイート文章 ##
        date_info = dict()
        image_filename = os.path.basename(image_filepath)
        date_info["year"] = image_filename[4:8]
        date_info["month"] = image_filename[9:11]
        date_info["day"] = image_filename[11:13]
        date_info["hour"] = image_filename[14:16]
        date_info["minute"] = image_filename[16:18]

        tweet = "{}年 {}月{}日 {}:{} の様子 #PlantRecord".format(
            date_info["year"],
            date_info["month"],
            date_info["day"],
            date_info["hour"],
            date_info["minute"])

        params = {"status" : tweet, "media_ids" : [media_ids]}

        res = twitter.post(url, params=params) #post送信

        if res.status_code == 200: #正常投稿出来た場合
            print("Success.")
        else: #正常投稿出来なかった場合
            print("Failed. : %d"% res.status_code)

        # 投稿したURLを返す
        res_dict = json.loads(res.text)
        twitter_url = res_dict["entities"]["media"][0]["expanded_url"]

        return twitter_url

    else:
        print("画像ファイルが存在しません")
        return None

if __name__ == "__main__":
    tweetInfo(image_filepath="images/pic_2021-0517-0218.jpg")


import numpy as np
import scipy.interpolate, scipy.linalg
import matplotlib.pyplot as plt
from numpy import array, cos, sin, pi, exp, log, log10, abs, max
from scipy.integrate import quad, odeint
from scipy.linalg import norm, solve

plt.ion()
plt.show()

## Question 1

f = lambda t: exp(t)/log(t)
df = lambda t: exp(t)/log(t)**2 * (log(t) - 1/t)
a, b = 2, 4
vm = []
vI = []

for k in range(1, 16):
    m = 2**k
    vt = np.linspace(a, b, m+1)
    h = 2**(1-k)
    I = 0
    for i in range(m):
        I += 2*f(vt[i]) + 4*f(vt[i+1]) - h*df(vt[i+1])
    vm.append(m)
    vI.append(h/6 * I)

I_exact = quad(f, a, b)[0]

vm = array(vm)
verr = abs(array(vI) - I_exact)

[p1, p0] = np.polyfit(log10(vm), log10(verr), 1)

plt.figure(0)
plt.clf()
plt.xlabel("m")
plt.ylabel("|I - I_m|")
plt.loglog(vm, 10**(p1*log10(vm) + p0), label="log |I - I_m| ~ a log m")
plt.loglog(vm, verr, linestyle='None', marker='o', label="Données")
plt.legend()
plt.title("Approximation de l'intégrale, a = {:.2f}".format(p1))

## Question 2

def enright(f, df, y0, vt):
    # y = np.asarray(y0).ravel()
    y = y0
    d = len(y)
    h = vt[1] - vt[0]
    vy = [y]
    for i in range(len(vt)-1):
        z = y
        err = 1
        while norm(err) > 1e-8:
            M = 2/3*h*df(z) - 1/6*h**2* df(y + h*f(y)) @ df(z) - np.eye(d)
            N = y + 1/3*h*f(y) + 2/3*h*f(z) - 1/6*h**2*df(y + h*f(y)) @ f(z) - z
            err = solve(M, N)
            z = z - err
        y = z
        vy.append(y)
    return array(vy)

## Question 3

f = lambda x: x*(1 - x)
df = lambda x: 1 - 2*x

y0 = array([0.1])
y_exact = np.vectorize(lambda t: y0 / (y0 - exp(-t)*(y0 - 1)))
t0, tf = 0, 2

vh = []
verr = []
for k in range(3, 13):
    h = 2**-k
    vt = np.arange(t0, tf+h, h)
    vy = enright(f, df, y0, vt)[:,0]
    vy_exact = y_exact(vt)
    err = max(abs(vy - vy_exact))
    vh.append(h)
    verr.append(err)

vh, verr = array(vh), array(verr)
[p1, p0] = np.polyfit(log10(vh), log10(verr), 1)

plt.figure(1)
plt.clf()
plt.xlabel("h")
plt.ylabel("|I - I_h|")
plt.loglog(vh, 10**(p1*log10(vh) + p0), label="log |I - I_h| ~ a log h")
plt.loglog(vh, verr, linestyle='None', marker='o', label="Données")
plt.legend()
plt.title("Modèle logistique avec Enright, a = {:.2f}".format(p1))

## Question 4

a = 30
f = lambda x: array([x[1], -x[0] + a*(1 - x[0]**2)*x[1]])
df = lambda x: array([[0, 1], [-1 + -2*a*x[0]*x[1], a*(1 - x[0]**2)]])

y0 = array([1, 0])
t0, tf = 0, 5*a
vy_exact = odeint(lambda y,t: f(y), y0, np.arange(t0, tf+0.001, 0.001))
vy = enright(f, df, y0, np.arange(t0, tf+0.01, 0.01))

plt.figure(2)
plt.clf()
plt.xlabel("theta(t)")
plt.ylabel("theta'(t)")
plt.plot(vy_exact[:,0], vy_exact[:,1], label="odeint")
plt.plot(vy[:,0], vy[:,1], label="enright", linestyle='None', marker='o', markersize=2)
plt.legend()
plt.title("Van der Pol avec Enright")