import requests
from selenium import webdriver
import json
from .base_tread import Base_tr
import random
import time
import base64
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.keys import Keys
import os
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

class Registr_fb_tr(Base_tr):

    def giv_name(self):
        try:
            self.driver.get("http://ipinfo.io/json")
            el = self.driver.find_element_by_xpath("//pre")
            settings = json.loads(el.text)
            self.country = settings["country"]
            if  self.country == "UA":

                last_name = [
                    "Ковалэнко",
                    "Бондарэнко",
                    "Злэнко",
                    "Саенко",
                    "Семеночко",
                    "Шумэйко",
                    "Мацэйко",
                    "Ламэйко",
                    "Бутэйко",
                    "Потебэнько",
                    "Опэнько",
                    "Витребэнько",
                    "Галаенко",
                    "Мотриенко",
                    "Стратиенко",
                    "Петрэнко",
                    "Павлэнко",
                    "Шевчэнко",
                    "Евэнко",
                    "Майстрэнко"]
                random.shuffle(last_name)

                first_name = [
                    "Агрипина",
                    "Ада",
                    "Аделаїда",
                    "Аделіна",
                    "Адріана",
                    "Богдана",
                    "Богуслава",
                    "Болеслава",
                    "Вероніка",
                    "Вікторина",
                    "Вікторія",
                    "Віола",
                    "Віолетта",
                    "Георгіна",
                    "Глафіра",
                    "Гликерія",
                    "Євлалія",
                    "Євлампія",
                    "Євпраксія",
                    "Єлизавета",
                    "Єпистима",
                    "Інга",
                    "Інеса",
                    "Інна",
                    "Іраїда",
                    "Кароліна",
                    "Катерина",
                    "Кіра",
                    "Клавдія",
                    "Клара",
                    "Клементина",
                    "Корнелія",
                    "Ліліана",
                    "Лілія",
                    "Ліна",
                    "Лія",
                    "Лукерія",
                    "Лукія",
                    "Меланія",
                    "Мелетія",
                    "Мелітина",
                    "Милана",
                    "Милослава",
                    "Сільвія",
                    "Сніжана",
                    "Соломія",
                    "Софія",
                    "Станіслава",
                    "Стелла"

                ]
                random.shuffle(first_name)

            elif  self.country == "RU":
                first_name = ["Агафья",
                              "Агриппина",
                              "Акулина",
                              "Алевтина",
                              "Александра",
                              "Алина",
                              "Алла",
                              "Анастасия",
                              "Ангелина",
                              "Анжела",
                              "Анжелика",
                              "Анна",
                              "Антонина",
                              "Валентина",
                              "Валерия",
                              "Варвара",
                              "Иоанна",
                              "Ираида",
                              "Ирина",
                              "Зинаида",
                              "Злата",
                              "Зоя",
                              "Маргарита",
                              "Марина",
                              "Мария",
                              "Марфа",
                              "Матрёна",
                              "Мирослава"]

                random.shuffle(first_name)
                last_name = [
                    "Амурская",
                    "Ангельская",
                    "Анненская",
                    "Афанасьева",
                    "Афинская",
                    "Бабочкина",
                    "Багирова",
                    "Баженова",
                    "Белоградская",
                    "Белозёрская",
                    "Березина",
                    "Беркутова",
                    "Благовещенская",
                    "Богословская",
                    "Бриллиантова",
                    "Василькова",
                    "Византийская",
                    "Воскресенская",
                    "Гиацинтова",
                    "Гончарова",
                    "Городецкая",
                    "Залесская",
                    "Елисеева",
                    "Златовласова",
                    "Златопольская",
                    "Знаменская",
                    "Зорина",
                    "Игнатьева",
                    "Истомина",
                    "Каменская",
                    "Колосовская",
                    "Лаврентьева",
                    "Луговая",
                    "Лучинская",
                    "Майская",
                    "Малиновская",
                    "Нагорная",
                    "Никитина",
                    "Озерова",
                    "Островская",
                    "Рассказова",
                    "Родионова",
                    "Рябинина",
                    "Румянцева",
                    "Сапфирова",
                    "Серебрянская",
                    "Солнцева",
                    "Ушанская",
                    "Цветкова",

                ]

                random.shuffle(last_name)

            else:
                first_name = ["Adriana", "Ada", "Abby", "Adrianne", "Aimee", "Beth", "Catherine", "Cathy", "Cheryl",
                              "ringo", "Jamie", "nicky", "Diane", "Donna", "trudie", "Esther", "lauren",
                              "ichabod", "arthur", "ashley", "Fiona", "julio", "lorraine", "floyd", "janet",
                              "lydia", "Irene", "Jasmine", "bradley"]
                random.shuffle(first_name)

                last_name = ["Smith", "Johnson", "Williams", "Jones", "Brown", "Davis", "Miller",
                             "Wilson", "Moore", "Wilson", "Taylor", "Anderson", "Thomas", "Jackson", "White"]
                random.shuffle(last_name)

            self.f_name = first_name[0]
            self.l_name = last_name[0]
        except:
            return (None, None)

        return (self.f_name, self.l_name)

    def form_filling(self):

        xpath = "//a[@class='_42ft _4jy0 _6lti _4jy6 _4jy2 selected _51sy']"
        self.click_to_xpath(xpath,circle=10, time_sleep=1)

        xpath = "//input[@name='firstname']"
        if not self.set_text(xpath, self.f_name):
            return (False)
        time.sleep(4)

        xpath = "//input[@name='lastname']"
        if not self.set_text(xpath, self.l_name):
            return (False)

        time.sleep(4)
        xpath = "//input[@name='reg_email__']"
        if not self.set_text(xpath, self.settings["email_f"]):
            return (False)

        time.sleep(4)
        xpath = "//input[@name='reg_email_confirmation__']"
        if not self.set_text(xpath,  self.settings["email_f"]):
            return (False)

        time.sleep(4)
        xpath = "//input[@name='reg_passwd__']"
        if not self.set_text(xpath, self.settings["fb_password"]):
            return (False)

        time.sleep(4)
        index = random.randint(2, 27)
        xpath = "//select[@name='birthday_day']"
        if not self.set_current_index(xpath, index):
            self.answer = {"answer": False, "comment": "Не удалось выбрать день рождения"}
            return (False)

        time.sleep(2)
        index = random.randint(1, 10)
        xpath = "//select[@name='birthday_month']"
        if not self.set_current_index(xpath, index):
            self.answer["status"] = "Ошибка"
            self.answer["comment"] = f"Не удалось выбрать месяц рождения"
            return (False)

        time.sleep(2)
        index = random.randint(20, 30)
        xpath = "//select[@name='birthday_year']"
        if not self.set_current_index(xpath, index):
            self.answer["status"] = "Ошибка"
            self.answer["comment"] = f"Не удалось выбрать год рождения"
            return (False)

        time.sleep(2)
        index = 0
        xpath = "//input[@name='sex']"
        if not self.selected_chec_boxs(xpath, index):
            self.answer["status"] = "Ошибка"
            self.answer["comment"] = f"Не удалось выбрать пол"
            return (False)

        time.sleep(2)
        xpath = "//button[@name='websubmit']"
        if not self.click_to_xpath(xpath):
            return (False)

        time.sleep(3)
        xpath = "//div[@id='reg_error_inner']"
        try:
            element = self.driver.find_element_by_xpath(xpath)
            height = element.size["height"]
            if height > 0:
                self.click_to_xpath(xpath)
        except:
            pass
        time.sleep(3)

        # elements = self.driver.find_elements_by_xpath(xpath)
        # if len(elements) > 0:
        #     self.click_to_xpath(xpath)

        # time.sleep(3)
        # elements = self.driver.find_elements_by_xpath(xpath)
        # if len(elements) > 0:

        xpath = "//div[@id='reg_error_inner']"
        try:
            element = self.driver.find_element_by_xpath(xpath)
            height = element.size["height"]
            if height > 0:

                self.answer["status"] = "Ошибка"
                self.answer["comment"] = f"Попытайтесь зарегестрироваться позже"
                return False
        except:
            pass

        return True

    def google_form_filling(self):
        xpath = "//button[@class='_42ft _4jy0 _4jy4 _4jy1 selected _51sy']"
        if self.click_to_xpath(xpath):
            time.sleep(10)
            window_after = self.driver.window_handles[1]
            self.driver.switch_to.window(window_after)

            if not self.click_to_xpath("//li[@class='M8HEDc ibdqA cd29Sd bxPAYd W7Aapd SmR8 znIWoc']"):

                xpath = "//input[@type='email']"
                if not self.set_text(xpath, self.settings["email_f"]):
                    self.answer["status"] = "Ошибка"
                    self.answer["comment"] = f"Время исчерпано, не удалось найти {xpath}"
                    return

                xpath = "//div[@id='identifierNext']"
                if not self.click_to_xpath(xpath):
                    return

                xpath = "//input[@type='password']"
                if not self.set_text(xpath, self.settings["email_password"]):
                    self.answer["status"] = "Ошибка"
                    self.answer["comment"] = f"Время исчерпано, не удалось найти {xpath}"
                    return

                xpath = "//div[@id='passwordNext']"
                if not self.click_to_xpath(xpath):
                    return

                xpath = "(//li[last()-1])[1]"
                if not self.click_to_xpath(xpath):
                    return

                elements = self.driver.find_elements_by_xpath("//button[@id='checkpointSubmitButton']")
                if len(elements) > 0:
                    self.answer["status"] = "Ошибка"
                    self.answer["comment"] = f"Аккаунт заблокированн"
                    return
                xpath = "//input[@id='identifierId']"
                if not self.set_text(xpath, self.settings["email_s"]):
                    self.answer["status"] = "Ошибка"
                    self.answer["comment"] = f"Время исчерпано, не удалось найти {xpath}"
                    return

                elements = self.driver.find_elements_by_xpath("//button[@id='checkpointSubmitButton']")
                if len(elements) > 0:
                    self.answer["status"] = "Ошибка"
                    self.answer["comment"] = f"Аккаунт заблокированн"
                    return
                xpath = "//input[@id='identifierId']"
                element = self.driver.find_element_by_xpath(xpath)
                element.send_keys(Keys.ENTER)

                elements = self.driver.find_elements_by_xpath("//button[@id='checkpointSubmitButton']")
                if len(elements) > 0:
                    self.answer["status"] = "Ошибка"
                    self.answer["comment"] = f"Аккаунт заблокированн"
                    return

            time.sleep(3)
            if len(self.driver.window_handles) > 1:
                window_after = self.driver.window_handles[1]
                self.driver.switch_to.window(window_after)
                self.driver.close()
                self.driver.switch_to.window(self.driver.window_handles[0])

            for i in range(3):
                webdriver.ActionChains(self.driver).send_keys(Keys.ESCAPE).perform()
                time.sleep(1)

            try:
                xpath = "//input[@name='ok']"
                self.click_to_xpath(xpath)
            except:
                pass

            xpath = "(//div[@id='bluebar_profile_and_home']//div//div)[1]"
            self.click_to_xpath(xpath)
        return True

    def load_img(self):
        # xpath = "(//div[@class='linkWrap noCount'])[1]"
        # if not self.click_to_xpath(xpath):
        #     self.answer["status"] = "Ошибка"
        #     self.answer["comment"] = f"Время исчерпано, не удалось найти {xpath}"
        #     return (False)

        if "face_picture_href" in self.settings and self.settings["face_picture_href"] != False:
            self.scroll_page_up(10)
            self.face_picture = f"face_picture_{self.settings['id_scenarios']}.jpg"
            self.face_picture_href = self.settings["face_picture_href"]
            try:
                os.remove(self.face_picture)
            except:
                pass

            p = requests.get(self.face_picture_href)
            out = open(self.face_picture, "wb")
            out.write(p.content)
            out.close()

            xpath = "//div[contains(@class, 'fbTimelineProfilePicSelector')]"
            if not self.click_to_xpath(xpath):
                return (False)

            for i in range(40):
                try:
                    elem = self.driver.find_element_by_xpath("//a[@class='_3cia']/div[@class='_3jk']/input[@type='file']")
                    elem.send_keys(self.face_picture)
                    break
                except:
                    time.sleep(1)

            time.sleep(10)

            xpath = "//button[@data-testid='profilePicSaveButton']"
            if not self.click_to_xpath(xpath):
                return False

            try:
                os.remove(self.face_picture)
            except:
                pass

            time.sleep(5)
            for i in range(3):
                webdriver.ActionChains(self.driver).send_keys(Keys.ESCAPE).perform()
                time.sleep(1)

            self.scroll_page_up(15)
        else:
            self.face_picture = None

        if "cover_picture_href" in self.settings and self.settings["cover_picture_href"] != False:
            self.cover_picture = f"cover_picture_{self.settings['id_scenarios']}.jpg"
            self.cover_picture_href = self.settings["cover_picture_href"]
            try:
                os.remove(self.cover_picture)
            except:
                pass

            p = requests.get(self.cover_picture_href)
            out = open(self.cover_picture, "wb")
            out.write(p.content)
            out.close()

            xpath = "//div[@id='fbProfileCoverPhotoSelector']"
            element_to_hover_over = self.driver.find_element_by_xpath(xpath)
            hover = ActionChains(self.driver).move_to_element(element_to_hover_over)
            hover.perform()

            if not self.click_to_xpath(xpath):
                return (False)

            time.sleep(3)
            elem = self.driver.find_element_by_xpath("//label//div[@class='_3jk']//input")
            elem.send_keys(self.cover_picture)
            time.sleep(10)

            xpath = "//button[@name='save']"
            if not self.click_to_xpath(xpath):
                return (False)

            try:
                os.remove(self.cover_picture)
            except:
                pass

        else:
            self.cover_picture = None

        return

    def login_mail(self):
        type_mail = self.settings["email_f"].split("@")[1]

        if type_mail == "yahoo.com":

            self.driver.get("https://login.yahoo.com/config/login?.src=fpctx&.intl=us&.lang=en-US&.done=https%3A%2F%2Fwww.yahoo.com")

            time.sleep(4)

            xpath = "//input[@id='login-username']"
            if not self.set_text(xpath, self.settings["email_f"]):
                return False
            else:
                self.enter_click(xpath)

            time.sleep(4)

            xpath = "//input[@id='login-passwd']"
            if not self.set_text(xpath, self.settings["email_password"]):
                return False

            else:
                self.enter_click(xpath)

            self.driver.get("https://mail.yahoo.com")

            time.sleep(5)
            if "mail.yahoo.com/d/folders/1" not in  self.driver.current_url:
                return False

        elif type_mail == "gmail.com":

            self.driver.get(
                "https://accounts.google.com/signin/v2/identifier?continue=https%3A%2F%2Fmail.google.com%2Fmail%2F&service=mail&sacu=1&rip=1&flowName=GlifWebSignIn&flowEntry=ServiceLogin")

            u = ["https://policies.google.com/privacy?gl=TM&hl=en",
                 "https://mail.google.com/mail/u/0/#inbox",
                 "https://mail.google.com/mail/#inbox",
                 "https://policies.google.com/privacy?gl=US&hl=ru",
                 "https://mail.google.com/mail/u/0/"]

            if not self.driver.current_url in u:

                xpath = "//div[@id='profileIdentifier']"
                if self.check_xpath(xpath):
                    xpath = "//input[@name='password']"
                    self.set_text(xpath, self.settings["email_password"])
                    self.enter_click(xpath)

                time.sleep(3)
                xpath = "//input[@id='identifierId']"
                if not self.set_text(xpath, self.settings["email_f"]):
                    return False
                else:
                    self.enter_click(xpath)

                time.sleep(4)
                xpath = "//input[@name='password']"
                if not self.set_text(xpath, self.settings["email_password"]):
                    return False
                else:
                    self.enter_click(xpath)

            time.sleep(5)

            self.driver.switch_to_window(self.driver.window_handles[0])

            xpath = "//li[last()-1]"
            if self.click_to_xpath(xpath,circle=5, time_sleep=1):
                xpath = "//input[@id='knowledge-preregistered-email-response']"
                self.set_text(xpath, self.settings["email_s"])
                self.enter_click(xpath)

            time.sleep(2)

            url = "accounts.google.com/signin/newfeatures"
            if url in self.driver.current_url:
                xpath = "//div[@class='U26fgb XHsn7e dURtfb Tk2jV']"
                self.click_to_xpath(xpath)
                xpath ="//div[@class='U26fgb XHsn7e dURtfb Tk2jV M9Bg4d']"
                for i in range(2):
                    self.click_to_xpath(xpath)

                xpath = "//div[@class='U26fgb O0WRkf zZhnYe e3Duub C0oVfc WVqvne ioikHf TMcGbb']"

                self.click_to_xpath(xpath)

            time.sleep(5)

            if not self.driver.current_url in u:
                return False

        else:
            return False

        return True

    def confirm_email(self):

        type_mail = self.settings["email_f"].split("@")[1]

        if type_mail == "yahoo.com":

            self.driver.get("https://mail.yahoo.com")

            xpath = "//button[@data-test-id='themes-cue-close']"
            self.click_to_xpath(xpath,circle=10, time_sleep=1)

            xpath = """(//a[@aria-label="Facebook's email"])[position() = last()]"""
            if self.click_to_xpath(xpath,circle=10, time_sleep=1):

                xpath = "//table/tbody/tr[3]/td[2]/table/tbody/tr[2]/td[3]/a"
                self.click_to_xpath(xpath,circle=10, time_sleep=1)
                return True

        elif type_mail == "gmail.com":

            self.driver.get("https://mail.google.com")

            xpath = "//button[@name='welcome_dialog_next']"
            self.click_to_xpath(xpath,circle=5, time_sleep=1)

            xpath = "//button[@name='ok']"
            self.click_to_xpath(xpath,circle=5, time_sleep=1)

            xpath = "(//span[@email='registration@facebookmail.com'])[position() = last()]/../../.."
            try:
                WebDriverWait(self.driver, 30).until(EC.presence_of_element_located((By.XPATH, xpath)))
                self.click_to_xpath(xpath)
                xpath = "//span[@style='font-family:Helvetica Neue,Helvetica,Lucida Grande,tahoma,verdana,arial,sans-serif;white-space:nowrap;font-weight:bold;vertical-align:middle;color:#ffffff;font-size:14px;line-height:14px']"
                self.click_to_xpath(xpath)
                time.sleep(3)
                return True
            except:
                return False
        else:
            return False

        return False

    def answers_questions(self):
        #where did you go to university?

        ua_ans = {  "which city do you live in?": ["Kiev", "Dnipro", "Kharkov", "Odessa"],
                    "what city are you from?":["Kiev", "Dnipro", "Kharkov", "Odessa"],
                    "where did you go to high school?":["киев школа 54",
                                                "киев школа 79",
                                                "киев школа 313",
                                                "киев школа 252",
                                                "киев школа 88",
                                                "киев школа 302",
                                                "киев школа 3",
                                                "киев школа 235",
                                                "киев школа 204",
                                                "киев школа 207"],
                    "where did you go to college?":["Львовский национальный университет им. Ивана Франка",
                                            "Национальный технический университет «Харьковский политехнический институт»",
                                            "Сумской государственный университет",
                                            "Киевский национальный университет им. Шевченко",
                                            "Харьковский национальный университет им. Каразина."],
                    "where do you work?":["Self-employed",
                                "student",
                                "retired"],
                    "где вы живете?":["Kiev", "Dnipro", "Kharkov", "Odessa"],
                    "из какого вы города?":["Kiev", "Dnipro", "Kharkov", "Odessa"],
                    "в какой школе вы учились?":["киев школа 54",
                                                "киев школа 79",
                                                "киев школа 313",
                                                "киев школа 252",
                                                "киев школа 88",
                                                "киев школа 302",
                                                "киев школа 3",
                                                "киев школа 235",
                                                "киев школа 204",
                                                "киев школа 207"],
                    "в каком вузе вы учились?":["Львовский национальный университет им. Ивана Франка",
                                            "Национальный технический университет «Харьковский политехнический институт»",
                                            "Сумской государственный университет",
                                            "Киевский национальный университет им. Шевченко",
                                            "Харьковский национальный университет им. Каразина."],
                    "где вы работаете?":["Self-employed",
                                "student",
                                "retired"],
                    "У якому місті ви живете?":["Kiev", "Dnipro", "Kharkov", "Odessa"],
                    "в якому місті Ви мешкаєте?":["Kiev", "Dnipro", "Kharkov", "Odessa"],
                    "У якій школі ви вчилися?":["киев школа 54",
                                                "киев школа 79",
                                                "киев школа 313",
                                                "киев школа 252",
                                                "киев школа 88",
                                                "киев школа 302",
                                                "киев школа 3",
                                                "киев школа 235",
                                                "киев школа 204",
                                                "киев школа 207"],
                    "Який коледж ви відвідували?":["Львовский национальный университет им. Ивана Франка",
                                            "Национальный технический университет «Харьковский политехнический институт»",
                                            "Сумской государственный университет",
                                            "Киевский национальный университет им. Шевченко",
                                            "Харьковский национальный университет им. Каразина."],
                    "Де ви працюєте?":["Self-employed",
                                "student",
                                "retired"]}

        ru_ans = {
                    "which city do you live in?": ["Moscow Russia Moscow"],
                    "what city are you from?":["Moscow Russia Moscow"],
                    "where did you go to high school?":["москва школа 303",
                                                "москва школа 124",
                                                "москва школа 252",
                                                "москва школа 2025",
                                                "москва школа 54",
                                                "москва школа 42",
                                                "москва школа 12"],
                    "where did you go to college?":["Львовский национальный университет им. Ивана Франка",
                                                "Национальный технический университет «Харьковский политехнический институт»",
                                                "Сумской государственный университет",
                                                "Киевский национальный университет им. Шевченко",
                                                "Харьковский национальный университет им. Каразина."],
                    "where do you work?":["Self-employed","student","retired"],
                    "what is your position at retired":["retiree"],
                    "где вы живете?":["Moscow","Saint-Peterburg","Omsk"],
                    "из какого вы города?":["Moscow","Saint-Peterburg","Omsk"],
                    "в какой школе вы учились?":["москва школа 303",
                                                "москва школа 124",
                                                "москва школа 252",
                                                "москва школа 2025",
                                                "москва школа 54",
                                                "москва школа 42",
                                                "москва школа 12"],
                    "в каком вузе вы учились?":["Львовский национальный университет им. Ивана Франка",
                                                "Национальный технический университет «Харьковский политехнический институт»",
                                                "Сумской государственный университет",
                                                "Киевский национальный университет им. Шевченко",
                                                "Харьковский национальный университет им. Каразина."],
                    "где вы работаете?":["Self-employed","student","retired"],
                    "У якому місті ви живете?":["Moscow Russia Moscow"],
                    "в якому місті Ви мешкаєте?":["Moscow Russia Moscow"],
                    "У якій школі ви вчилися?":["москва школа 303",
                                                "москва школа 124",
                                                "москва школа 252",
                                                "москва школа 2025",
                                                "москва школа 54",
                                                "москва школа 42",
                                                "москва школа 12"],
                    "Який коледж ви відвідували?":["Львовский национальный университет им. Ивана Франка",
                                                    "Национальный технический университет «Харьковский политехнический институт»",
                                                    "Сумской государственный университет",
                                                    "Киевский национальный университет им. Шевченко",
                                                    "Харьковский национальный университет им. Каразина."],
                    "Де ви працюєте?":["Self-employed","student", "retired"]}

        if self.country == "UA":
            ans = ua_ans
        else :
            ans = ru_ans

        for i in range(5):
            time.sleep(5)
            for j in range(3):
                webdriver.ActionChains(self.driver).send_keys(Keys.ESCAPE).perform()
                time.sleep(1)

            self.scroll_page_up(10)
            xpath = "((//div[@id='profile_timeline_info_review']/li/div/div/div/div)[2]/div)[1]"
            try:
                element = self.driver.find_element_by_xpath(xpath)
                question = element.text
                question = question.split(', ')
            except:
                continue

            if len(question) > 1:
                question = question[1]
            else:
                question = question[0]

            question = question.lower()

            if question not in ans:
                print(f"Не определено {question}")

                xpath = "//button[@value='deny']"
                self.click_to_xpath(xpath)

            else:
                for j in range(3):
                    webdriver.ActionChains(self.driver).send_keys(Keys.ESCAPE).perform()
                    time.sleep(1)
                answears = ans[question]
                random.shuffle(answears)
                my_answer = answears[0]

                xpath = "//input[@data-testid='searchable-text-input']"
                self.set_text(xpath, my_answer)

                xpath = "//li[@data-testid='choose_contact_button']"
                self.click_to_xpath(xpath)

                xpath = "//input[@data-testid='searchable-text-input']"
                self.enter_click(xpath)

                xpath = "//button[@value='confirm']"
                self.click_to_xpath(xpath)

    def add_groups(self):
        try:
            time.sleep(3)
            self.driver.switch_to_window(self.driver.window_handles[1])
            time.sleep(3)
            self.driver.refresh()
        except:
            return

        if self.country == "RU":
            file_groups = "RU group.txt"

        elif self.country == "UA":
            file_groups = "UA group.txt"

        else:
            file_groups = "general.txt"

        links = []
        f = open(file_groups, "r")

        for link in f:
            if link == "\n":
                continue

            links.append(link.replace("\n",""))

        random.shuffle(links)

        for i in range(2):
            link = links[i]
            self.driver.get(link)

            for j in range(3):
                webdriver.ActionChains(self.driver).send_keys(Keys.ESCAPE).perform()
                time.sleep(1)

            xpath = "//button[@data-testid='page_profile_like_button_test_id']"
            self.click_to_xpath(xpath)

    def run(self):
        self.cover_picture = None
        self.face_picture = None
        answer = self.connect_to_multilogin()
        if not answer["answer"]:
            self.answer["status"] = "Ошибка"
            self.answer["comment"] = answer["comment"]
            try:
                self.driver.quit()
            except:
                pass
            return

        if not self.login_mail():
            self.answer["status"] = "Ошибка"
            self.answer["comment"] = f"Не удалось зайти на почту"
            self.driver.quit()
            return

        self.giv_name()

        link = 'https://www.facebook.com'
        if not self.get_tu_link(link):
            self.answer["status"] = "Ошибка"
            self.answer["comment"] = f"Не удалось зайти на https://www.facebook.com"
            self.driver.quit()
            return

        time.sleep(3)

        if not self.check_block():
            self.answer["status"] = "Ошибка"
            self.answer["comment"] = f"Аккаунт заблокирован"
            self.driver.quit()
            return

        for i in range(3):
            webdriver.ActionChains(self.driver).send_keys(Keys.ESCAPE).perform()
            time.sleep(1)

        xpath = "//span[@class='_2md']"
        if self.find_xpath(xpath, circle=5, time_sleep=1) and  not "www.facebook.com/confirmemail.php" in self.driver.current_url :
                self.answer["status"] = "Ошибка"
                self.answer["comment"] = f"Аккаунт уже зарегистрированн"
                self.driver.quit()
                return

        elif not "www.facebook.com/confirmemail.php" in self.driver.current_url:
            if not self.form_filling():
                self.driver.quit()
                return

        for i in range(3):
            webdriver.ActionChains(self.driver).send_keys(Keys.ESCAPE).perform()
            time.sleep(1)

        if not self.check_block():
            self.answer["status"] = "Ошибка"
            self.answer["comment"] = f"Аккаунт заблокирован после заполнения формы"
            self.driver.quit()
            return

        if  not "www.facebook.com/confirmemail.php" in self.driver.current_url :
                self.answer["status"] = "Ошибка"
                self.answer["comment"] = f"Аккаунт уже зарегистрированн"
                self.driver.quit()
                return

        xpath = "//*[@id='bluebar_profile_and_home']/div/div/a"
        self.click_to_xpath(xpath)

        for i in range(4):
            webdriver.ActionChains(self.driver).send_keys(Keys.ESCAPE).perform()
            time.sleep(1)

        self.load_img()

        self.answers_questions()

        if self.cover_picture == None:
            cover_picture = "caver не залит"
        else:
            cover_picture = "caver залит"

        if self.face_picture == None:
            face_picture = " face не залит"
        else:
            face_picture = "face залит"

        if not self.confirm_email():
            email_comment = "почта не подтверждена"
        else:
            email_comment = "почта подтверждена"
            self.add_groups()

        self.answer["status"] = "Выполнен"
        self.answer["comment"] = f"Аккаунт зарегистрирован в фб {email_comment,face_picture,cover_picture}"
        self.driver.quit()


# Import Libraries
import numpy as np
import tensorflow as tf
# from datetime import datetime
import matplotlib.pyplot as plt
import h5py
from tensorflow.keras.layers import Input, Dense, Conv2D, MaxPool2D, AveragePooling2D, UpSampling2D, concatenate, \
    BatchNormalization, Conv2DTranspose, Flatten, Reshape
from tensorflow.keras.optimizers import Adam

import os
os.environ["KMP_DUPLICATE_LIB_OK"]="TRUE"

# -------------------------------------------------------------------------------------------------
# FILE SELECTION
Re = 20.0  # choose between Re= 20.0, 30.0, 40.0, 50.0, 60.0, 100.0, 180.0

# T has different values depending on Re
if Re == 20.0 or Re == 30.0 or Re == 40.0:
    T = 20000
else:
    T = 2000

path_folder = 'Final_Product/'  # path to folder in which flow data is situated
path = path_folder + f'Kolmogorov_Re{Re}_T{T}_DT01.h5'

# -------------------------------------------------------------------------------------------------
# PARAMETER SELECTION

dimensions = [8, 4, 2, 1]  # Specify the dimensions for each encoding layer (dim[3] = dimension of latent space)
# e.g. [16,8,4,2], [32,16,8,4], [64,32,16,8], [128,64,32,16], [256, 128, 64, 32]
# Decoding layer is mirror of encoding layer

activation_function = 'tanh'  # specify which activation function to use
learning_rate = 0.1  # change to 0.0001 if using a latent space dimension of 32 and 64
number_epochs = 10  # set high, early stopping function will stop iterations if it sees no visible change after 'patience_early_stopping' number of epochs
patience_early_stopping = 20  # how many epochs with no significant change before AE stops training
pooling = 'max'  # set to 'max' or 'average'
if pooling == 'max':
    pooling_function = MaxPool2D
else:
    pooling_function = AveragePooling2D

# -------------------------------------------------------------------------------------------------
# READ DATASET
hf = h5py.File(path, 'r')
Nx = 24
Nu = 1
t = np.array(hf.get('t'))
u_all = np.zeros((Nx, Nx, len(t), Nu))
u_all[:, :, :, 0] = np.array(hf.get('u_refined'))
# u_all[:,:,:,1] = np.array(hf.get('v_refined'))
u_all = np.transpose(u_all, [2, 0, 1, 3])
hf.close()
print(u_all.shape)
print('Read Dataset')

# normalize data
u_min = np.amin(u_all[:, :, :, 0])
u_max = np.amax(u_all[:, :, :, 0])
u_all[:, :, :, 0] = (u_all[:, :, :, 0] - u_min) / (u_max - u_min)
if Nu == 2:
    v_min = np.amin(u_all[:, :, :, 1])
    v_max = np.amax(u_all[:, :, :, 1])
    u_all[:, :, :, 1] = (u_all[:, :, :, 1] - v_min) / (v_max - v_min)
print('Normalized Data')

# visualization of the dataset
fig = plt.figure()
ax = fig.add_subplot(121)
ax.contourf(u_all[50, :, :, 0])
# ax2 = fig.add_subplot(122)
# ax2.contourf(u_all[0,:,:,1])
plt.show()
print('Showed data')

# -------------------------------------------------------------------------------------------------
# PREPARE DATASET
val_ratio = int(np.round(0.75 * len(u_all)))
test_ratio = int(np.round(0.95 * len(u_all)))

u_train = u_all[:val_ratio, :, :, :].astype('float32')
u_val = u_all[val_ratio:test_ratio, :, :, :].astype('float32')
u_test = u_all[test_ratio:, :, :, :].astype('float32')
print('Prepared Dataset')

# -------------------------------------------------------------------------------------------------
# CREATING NETWORK
input_img = Input(shape=(Nx, Nx, Nu))

nb_layer = 0

x = Conv2D(dimensions[0], (3, 3), padding='same', activation=activation_function)(input_img)
x = pooling_function((2, 2), padding='same')(x)
x = Conv2D(dimensions[1], (3, 3), padding='same', activation=activation_function)(x)
x = pooling_function((2, 2), padding='same')(x)
x = Conv2D(dimensions[2], (3, 3), padding='same', activation=activation_function)(x)
x = pooling_function((2, 2), padding='same')(x)
x = Conv2D(dimensions[3], (3, 3), padding='same', activation=activation_function)(x)
encoded = pooling_function((3, 3), padding='same')(x)

x = Conv2DTranspose(dimensions[3], (3, 3), padding='same', activation=activation_function)(encoded)
x = UpSampling2D((3, 3))(x)
x = Conv2DTranspose(dimensions[2], (3, 3), padding='same', activation=activation_function)(x)
x = UpSampling2D((2, 2))(x)
x = Conv2DTranspose(dimensions[1], (3, 3), padding='same', activation=activation_function)(x)
x = UpSampling2D((2, 2))(x)
x = Conv2DTranspose(dimensions[0], (3, 3), padding='same', activation=activation_function)(x)
x = UpSampling2D((2, 2))(x)
decoded = Conv2DTranspose(Nu, (3, 3), activation='linear', padding='same')(x)

autoencoder = tf.keras.models.Model(input_img, decoded)
encoder = tf.keras.models.Model(input_img, encoded)
print('Encoder Created')

# definition of the decoder
encoded_input = Input(shape=(1, 1, encoded.shape[3]))
deco = autoencoder.layers[-7](encoded_input)  # we re-use the same layers as the ones of the autoencoder
for i in range(6):
    deco = autoencoder.layers[-6 + i](deco)

decoder = tf.keras.models.Model(encoded_input, deco)

print('Decoder Created')
print(encoder.summary())
print(decoder.summary())
print(autoencoder.summary())

# -------------------------------------------------------------------------------------------------
# TRAINING NETWORK
autoencoder.compile(optimizer=Adam(learning_rate=learning_rate), loss='mse')
checkpoint_filepath = './checkpoint'
model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
    filepath=checkpoint_filepath,
    save_weights_only=True,
    monitor='val_loss',
    mode='min',
    save_best_only=True)
early_stop_callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=patience_early_stopping)

nb_epochs = number_epochs
batch_size = 20
hist = autoencoder.fit(u_train, u_train, epochs=nb_epochs, batch_size=batch_size,
                       shuffle=True, validation_data=(u_test, u_test),
                       verbose=1,
                       callbacks=[model_checkpoint_callback, early_stop_callback])

loss_history = hist.history['loss']
val_history = hist.history['val_loss']
plt.plot(loss_history, 'b')
plt.plot(val_history, 'r')
plt.title("Loss History")
plt.xlabel("Epoch")
plt.ylabel("Loss")

plt.show()
print('Trained Model')
# -------------------------------------------------------------------------------------------------
# VISUALIZATION OF PREDICTION
y_nn = autoencoder.predict(u_test[0:1, :, :, :])
print(y_nn.shape)

fig = plt.figure()
ax = fig.add_subplot(121)
ax.contourf(y_nn[0, :, :, 0])
ax = fig.add_subplot(122)
ax.contourf(u_test[0, :, :, 0])
plt.show()

print(f'AE error: {loss_history[-1]}')


import os
from pathlib import Path

base_path = input('Please enter the path where you want to search for AAE Files: \n')
curr_folder = os.path.basename(base_path)
aae_folder_path = os.path.join(base_path,f'AAE Files ({curr_folder})')
# print(our_path)

entries = os.listdir(base_path)
# print(entries)

for file in entries:
    curr_file_path = os.path.join(base_path,file)
    # print(full_file_path)
    # print(Path(full_file_path).suffix.lower())
    # print(Path(full_file_path).is_file())
    # print(os.path.getsize(full_file_path)/(1024*1024))
    if Path(curr_file_path).suffix.lower()=='.aae' and Path(curr_file_path).is_file():
        # print(f'{file} is an AAE file.')
        if not os.path.exists(aae_folder_path):
            os.mkdir(aae_folder_path)
        new_file_path=os.path.join(aae_folder_path,file)
        os.rename(curr_file_path,new_file_path)
    with open(os.path.join(aae_folder_path,'AAE Files.txt'),'a+') as curr_log:
        curr_log.write(f'\nMoved AAE file - "{file}".')

# F:\Musical World\Music Videos\^ H ^\90's
# H:\Pics\# Timeline #\2022\August 2022\New folder (4)

My Projects/Scripts for Photos Management/Move AAE Files to AAE Files Folder.py


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Jul 16 00:11:32 2020

@author: lukepinkel
"""

import re
import patsy
import pandas as pd
import numpy as np # analysis:ignore
import scipy as sp # analysis:ignore
import scipy.sparse as sps # analysis:ignore
from ..utilities.linalg_operations import (dummy, vech, invech, _check_np,
                                           khatri_rao, sparse_woodbury_inversion,
                                           _check_shape, woodbury_inversion)
from ..utilities.special_mats import lmat, nmat
from ..utilities.numerical_derivs import so_gc_cd, so_fc_cd
from .families import (Binomial, ExponentialFamily, Gamma, Gaussian,  # analysis:ignore
                            InverseGaussian, Poisson, NegativeBinomial)

from ..utilities.output import get_param_table
from sksparse.cholmod import cholesky

def replace_duplicate_operators(match):
    return match.group()[-1:]

def parse_random_effects(formula):
    matches = re.findall("\([^)]+[|][^)]+\)", formula)
    groups = [re.search("\(([^)]+)\|([^)]+)\)", x).groups() for x in matches]
    frm = formula
    for x in matches:
        frm = frm.replace(x, "")
    fe_form = re.sub("(\+|\-)(\+|\-)+", replace_duplicate_operators, frm)
    return fe_form, groups

def construct_random_effects(groups, data, n_vars):
    re_vars, re_groupings = list(zip(*groups))
    re_vars, re_groupings = set(re_vars), set(re_groupings)
    Zdict = dict(zip(re_vars, [_check_np(patsy.dmatrix(x, data=data, return_type='dataframe')) for x in re_vars]))
    Jdict = dict(zip(re_groupings, [dummy(data[x]) for x in re_groupings]))
    dim_dict = {}
    Z = []
    for x, y in groups:
        Ji, Xi = Jdict[y], Zdict[x]
        dim_dict[y] = {'n_groups':Ji.shape[1], 'n_vars':Xi.shape[1]}
        Zi = khatri_rao(Ji.T, Xi.T).T
        Z.append(Zi)
    Z = np.concatenate(Z, axis=1)
    return Z, dim_dict

def construct_model_matrices(formula, data, return_fe=False):
    fe_form, groups = parse_random_effects(formula)
    yvars, fe_form = re.split("[~]", fe_form)
    fe_form = re.sub("\+$", "", fe_form)
    yvars = re.split(",", re.sub("\(|\)", "", yvars))
    yvars = [x.strip() for x in yvars]
    n_vars = len(yvars)
    Z, dim_dict = construct_random_effects(groups, data, n_vars)
    X = patsy.dmatrix(fe_form, data=data, return_type='dataframe')
    fe_vars = X.columns
    y = data[yvars]
    X, y = _check_np(X), _check_np(y)
    if return_fe:
        return X, Z, y, dim_dict, list(dim_dict.keys()), fe_vars
    else:
        return X, Z, y, dim_dict, list(dim_dict.keys())

def make_theta(dims):
    theta, indices, index_start = [], {}, 0
    dims = dims.copy()
    dims['error'] = dict(n_groups=0, n_vars=1)
    for key, value in dims.items():
        n_vars = value['n_vars']
        n_params = int(n_vars * (n_vars+1) //2)
        indices[key] = np.arange(index_start, index_start+n_params)
        theta.append(vech(np.eye(n_vars)))
        index_start += n_params
    theta = np.concatenate(theta)
    return theta, indices

def make_gcov(theta, indices, dims, inverse=False):
    Gmats, g_indices, start = {}, {}, 0
    for key, value in dims.items():
        dims_i = dims[key]
        ng, nv = dims_i['n_groups'],  dims_i['n_vars']
        nv2, nvng = nv*nv, nv*ng
        theta_i = theta[indices['theta'][key]]
        if inverse:
            theta_i = np.linalg.inv(invech(theta_i)).reshape(-1, order='F')
        else:
            theta_i = invech(theta_i).reshape(-1, order='F')
        row = np.repeat(np.arange(nvng), nv)
        col = np.repeat(np.arange(ng)*nv, nv2)
        col = col + np.tile(np.arange(nv), nvng)
        data = np.tile(theta_i, ng)
        Gmats[key] = sps.csc_matrix((data, (row, col)))
        g_indices[key] = np.arange(start, start+len(data))
        start += len(data)
    G = sps.block_diag(list(Gmats.values())).tocsc()
    return G, g_indices

def lndet_gmat(theta, dims, indices):
    lnd = 0.0
    for key, value in dims.items():
        dims_i = dims[key]
        ng = dims_i['n_groups']
        Sigma_i = invech(theta[indices['theta'][key]])
        lnd += ng*np.linalg.slogdet(Sigma_i)[1]
    return lnd

def lndet_gmat_chol(theta, dims, indices):
    lnd = 0.0
    for key, value in dims.items():
        dims_i = dims[key]
        ng = dims_i['n_groups']
        theta_i = theta[indices['theta'][key]]
        L_i = invech_chol(theta_i)
        Sigma_i = L_i.dot(L_i.T)
        lnd += ng*np.linalg.slogdet(Sigma_i)[1]
    return lnd

def invech_chol(lvec):
    p = int(0.5 * ((8*len(lvec) + 1)**0.5 - 1))
    L = np.zeros((p, p))
    a, b = np.triu_indices(p)
    L[(b, a)] = lvec
    return L

def transform_theta(theta, dims, indices):
    for key in dims.keys():
        G = invech(theta[indices['theta'][key]])
        L = np.linalg.cholesky(G)
        theta[indices['theta'][key]] = vech(L)
    return theta

def inverse_transform_theta(theta, dims, indices):
    for key in dims.keys():
        L = invech_chol(theta[indices['theta'][key]])
        G = L.dot(L.T)
        theta[indices['theta'][key]] = vech(G)
    return theta

def get_d2_chol(dim_i):
    p = dim_i['n_vars']
    Lp = lmat(p).A
    T = np.zeros((p, p))
    H = []
    Ip = np.eye(p)
    for j, i in list(zip(*np.triu_indices(p))):
        T[i, j] = 1
        Hij = (Lp.dot(np.kron(Ip, T+T.T)).dot(Lp.T))[np.newaxis]
        H.append(Hij)
        T[i, j] = 0
    H = np.concatenate(H, axis=0)
    return H

def get_jacmats2(Zs, dims, indices, g_indices, theta):
    start = 0
    jac_mats = {}
    for key, value in dims.items():
        nv, ng =  value['n_vars'], value['n_groups']
        jac_mats[key] = []
        Zi = Zs[:, start:start+ng*nv]
        theta_i = theta[indices[key]]
        nv2, nvng = nv*nv, nv*ng
        row = np.repeat(np.arange(nvng), nv)
        col = np.repeat(np.arange(ng)*nv, nv2)
        col = col + np.tile(np.arange(nv), nvng)
        for i in range(len(theta_i)):
            dtheta_i = np.zeros_like(theta_i)
            dtheta_i[i] = 1.0
            dtheta_i = invech(dtheta_i).reshape(-1, order='F')
            data = np.tile(dtheta_i, ng)
            dGi = sps.csc_matrix((data, (row, col)))
            dVi = Zi.dot(dGi).dot(Zi.T)
            jac_mats[key].append(dVi)
        start+=ng*nv
    jac_mats['error'] = [sps.eye(Zs.shape[0])]
    return jac_mats

class LMEC:

    def __init__(self, formula, data, weights=None):

        indices = {}
        X, Z, y, dims, levels, fe_vars = construct_model_matrices(formula, data, return_fe=True)
        theta, theta_indices = make_theta(dims)

        indices['theta'] = theta_indices

        G, g_indices = make_gcov(theta, indices, dims)

        indices['g'] = g_indices

        XZ, Xty, Zty, yty = np.hstack([X, Z]), X.T.dot(y), Z.T.dot(y), y.T.dot(y)

        C, m = sps.csc_matrix(XZ.T.dot(XZ)), sps.csc_matrix(np.vstack([Xty, Zty]))
        M = sps.bmat([[C, m], [m.T, yty]])
        M = M.tocsc()
        self.fe_vars = fe_vars
        self.X, self.Z, self.y, self.dims, self.levels = X, Z, y, dims, levels
        self.XZ, self.Xty, self.Zty, self.yty = XZ, Xty, Zty, yty
        self.C, self.m, self.M = C, m, M
        self.theta, self.theta_chol = theta, transform_theta(theta, dims, indices)
        self.G = G
        self.indices = indices
        self.R = sps.eye(Z.shape[0])
        self.Zs = sps.csc_matrix(Z)
        self.jac_mats = get_jacmats2(self.Zs, self.dims, self.indices['theta'],
                                     self.indices['g'], self.theta)
        self.t_indices = list(zip(*np.triu_indices(len(theta))))
        self.elim_mats, self.symm_mats, self.iden_mats = {}, {}, {}
        self.d2g_dchol = {}
        for key in self.levels:
            p = self.dims[key]['n_vars']
            self.elim_mats[key] = lmat(p).A
            self.symm_mats[key] = nmat(p).A
            self.iden_mats[key] = np.eye(p)
            self.d2g_dchol[key] = get_d2_chol(self.dims[key])
        self.bounds = [(0, None) if x==1 else (None, None) for x in self.theta]
        self.bounds_2 = [(1e-6, None) if x==1 else (None, None) for x in self.theta]

    def update_mme(self, Ginv, s):
        """
        Parameters
        ----------
        Ginv: sparse matrix
             scipy sparse matrix with inverse covariance block diagonal

        s: scalar
            error covariance

        Returns
        -------
        M: sparse matrix
            updated mixed model matrix

        """
        M = self.M.copy()/s
        M[-Ginv.shape[0]-1:-1, -Ginv.shape[0]-1:-1] += Ginv
        return M

    def update_gmat(self, theta, inverse=False):
        """
        Parameters
        ----------
        theta: ndarray
             covariance parameters on the original scale

        inverse: bool
            whether or not to inverse G

        Returns
        -------
        G: sparse matrix
            updated random effects covariance

        """
        G = self.G
        for key in self.levels:
            ng = self.dims[key]['n_groups']
            theta_i = theta[self.indices['theta'][key]]
            if inverse:
                theta_i = np.linalg.inv(invech(theta_i)).reshape(-1, order='F')
            else:
                theta_i = invech(theta_i).reshape(-1, order='F')
            G.data[self.indices['g'][key]] = np.tile(theta_i, ng)
        return G

    def loglike(self, theta, use_sw=False):
        """
        Parameters
        ----------

        theta: array_like
            The original parameterization of the model parameters

        Returns
        -------
        loglike: scalar
            Log likelihood of the model
        """
        Ginv = self.update_gmat(theta, inverse=True)
        M = self.update_mme(Ginv, theta[-1])
        logdetG = lndet_gmat(theta, self.dims, self.indices)
        L = np.linalg.cholesky(M.A)
        ytPy = np.diag(L)[-1]**2
        logdetC = np.sum(2*np.log(np.diag(L))[:-1])
        logdetR = np.log(theta[-1]) * self.Z.shape[0]
        ll = logdetR + logdetC + logdetG + ytPy
        return ll

    def gradient2(self, theta, use_sw=False):
        """
        Parameters
        ----------
        theta: array_like
            The original parameterization of the components

        Returns
        -------
        gradient: array_like
            The gradient of the log likelihood with respect to the covariance
            parameterization

        Notes
        -----
        This function has a memory requirement proportional to O(n^2), as
        a dense (n x n) matrix (P) needs to be formed.  For models with
        n>10000, it is generally both faster and more feasible to use
        gradient_me

        """
        Ginv = self.update_gmat(theta, inverse=True)
        Rinv = self.R / theta[-1]
        if use_sw:
            Vinv = sparse_woodbury_inversion(self.Zs, Cinv=Ginv, Ainv=Rinv.tocsc())
        else:
            Vinv = woodbury_inversion(self.Z, Cinv=Ginv.A, Ainv=Rinv.A)
        W = (Vinv.dot(self.X))
        XtW = W.T.dot(self.X)
        XtW_inv = np.linalg.inv(XtW)
        P = Vinv - np.linalg.multi_dot([W, XtW_inv, W.T])
        Py = P.dot(self.y)
        grad = []
        for key in (self.levels+['error']):
            for dVdi in self.jac_mats[key]:
                gi = np.einsum("ij,ji->", dVdi.A, P) - Py.T.dot(dVdi.dot(Py))
                grad.append(gi)
        grad = np.concatenate(grad)
        grad = _check_shape(np.array(grad))
        return grad

    def gradient(self, theta, use_sw=False):
        """
        Parameters
        ----------
        theta: array_like
            The original parameterization of the components

        Returns
        -------
        gradient: array_like
            The gradient of the log likelihood with respect to the covariance
            parameterization

        Notes
        -----
        This function has a memory requirement proportional to O(n^2), as
        a dense (n x n) matrix (P) needs to be formed.  For models with
        n>10000, it is generally both faster and more feasible to use
        gradient_me

        """
        G = self.update_gmat(theta, inverse=False)
        R = self.R * theta[-1]
        V = self.Zs.dot(G).dot(self.Zs.T) + R
        chol_fac = cholesky(V)
        if use_sw:
            Rinv = self.R / theta[-1]
            Ginv = self.update_gmat(theta, inverse=True)
            RZ = Rinv.dot(self.Zs)
            Q = Ginv + self.Zs.T.dot(RZ)
            Vinv = Rinv - RZ.dot(cholesky(Q).inv()).dot(RZ.T)
        else:
            Vinv = chol_fac.solve_A(sp.sparse.eye(V.shape[0], format='csc'))

        if Vinv.nnz / np.product(Vinv.shape) > 0.1:
            Vinv = Vinv.A
        W = chol_fac.solve_A(self.X)
        XtW = W.T.dot(self.X)
        U = np.linalg.solve(XtW, W.T)
        Py = chol_fac.solve_A(self.y) - W.dot(U.dot(self.y))

        grad = []
        for key in (self.levels+['error']):
            for dVdi in self.jac_mats[key]:
                VdVdi = dVdi.dot(Vinv).diagonal().sum()
                trPdV = VdVdi - np.einsum('ij,ji->', W,
                                          sp.sparse.csc_matrix.dot(U, dVdi))
                gi = trPdV - Py.T.dot(dVdi.dot(Py))
                grad.append(gi)
        grad = np.concatenate(grad)
        grad = _check_shape(np.array(grad))
        return grad

    def gradient_me(self, theta):
        """
        Parameters
        ----------
        theta: array_like
            The original parameterization of the components

        Returns
        -------
        gradient: array_like
            The gradient of the log likelihood with respect to the covariance
            parameterization

        Notes
        -----
        This function avoids forming the (n x n) matrix P, and instead takes
        advantage of the fact that yP(dV)Py can be computed using mostly matrix
        vector products, while tr(P(dV)) can be computed by accumulating
        n vector-vector products where each component of P, P_i, can be formed
        only when needed.
        """
        Ginv = self.update_gmat(theta, inverse=True)
        Rinv = self.R / theta[-1]
        X, Z, y = self.X, self.Zs, self.y
        W = Rinv.dot(Z)
        Omega = cholesky((Z.T.dot(W) + Ginv).tocsc()).inv()
#        U = Rinv - W.dot(Omega).dot(W.T)
        UX = Rinv.dot(X) - W.dot(Omega).dot(W.T.dot(X))
        Uy = Rinv.dot(y) - W.dot(Omega).dot(W.T.dot(y))
        self.jac_mats['error'] = [self.jac_mats['error'][0].tocsc()]
        S = X.T.dot(UX)
        Sinv = np.linalg.inv(S)
        Py = Uy - UX.dot(np.linalg.inv(S).dot(UX.T.dot(y)))
        UXS = UX.dot(Sinv)
        grad = np.zeros_like(theta)
        k=0
        for key in (self.levels+['error']):
             for dVdi in self.jac_mats[key]:
                 grad[k] += -Py.T.dot(dVdi.dot(Py))[0][0]
                 k+=1
        for i in range(y.shape[0]):
#            P_i = np.asarray(U[i] - UXS[i].dot(UX.T))
            P_i = np.asarray((Rinv.tocsc()[i].T - W.dot(Omega.dot(W[i].T))).A.T[0] - UXS[i].dot(UX.T))
            k=0
            for key in (self.levels+['error']):
                for dVdi in self.jac_mats[key]:
#                    grad[k] = grad[k] + dVdi[:, i].T.dot(P_i[0])[0]
                    grad[k] = grad[k] + dVdi[:, i].T.dot(P_i)[0]
                    k=k+1
        return grad

    def hessian(self, theta):
        Ginv = self.update_gmat(theta, inverse=True)
        Rinv = self.R / theta[-1]
        Vinv = sparse_woodbury_inversion(self.Zs, Cinv=Ginv, Ainv=Rinv.tocsc())
        W = (Vinv.dot(self.X))
        XtW = W.T.dot(self.X)
        XtW_inv = np.linalg.inv(XtW)
        P = Vinv - np.linalg.multi_dot([W, XtW_inv, W.T])
        Py = P.dot(self.y)
        H = []
        PJ, yPJ = [], []
        for key in (self.levels+['error']):
            J_list = self.jac_mats[key]
            for i in range(len(J_list)):
                Ji = J_list[i].T
                PJ.append((Ji.dot(P)).T)
                yPJ.append((Ji.dot(Py)).T)
        t_indices = self.t_indices
        for i, j in t_indices:
            PJi, PJj = PJ[i], PJ[j]
            yPJi, JjPy = yPJ[i], yPJ[j].T
            Hij = -(PJi.dot(PJj)).diagonal().sum()\
                        + (2 * (yPJi.dot(P)).dot(JjPy))[0]
            H.append(np.array(Hij[0]))
        H = invech(np.concatenate(H)[:, 0])
        return H

    def update_chol(self, theta, inverse=False):
        """
        Parameters
        ----------
        theta: array_like
            array containing the lower triangular components of the cholesky
            for each random effect covariance

        inverse: bool

        Returns
        -------
        L_dict: dict of array_like
            Dictionary whose keys and values correspond to level names
            and the corresponding cholesky of the level's random effects
            covariance

        """
        L_dict = {}
        for key in self.levels:
            theta_i = theta[self.indices['theta'][key]]
            L_i = invech_chol(theta_i)
            L_dict[key] = L_i
        return L_dict

    def dg_dchol(self, L_dict):
        """

        Parameters
        ----------

        L_dict: dict of array_like
            Dictionary whose keys and values correspond to level names
            and the corresponding cholesky of the level's random effects
            covariance

        Returns
        -------

        Jf: dict of array_like
            For each level contains the derivative of the cholesky parameters
            with respect to the covariance

        Notes
        -----

        Function evaluates the derivative of the cholesky parameterization
        with respect to the lower triangular components of the covariance

        """

        Jf = {}
        for key in self.levels:
            L = L_dict[key]
            E = self.elim_mats[key]
            N = self.symm_mats[key]
            I = self.iden_mats[key]
            Jf[key] = E.dot(N.dot(np.kron(L, I))).dot(E.T)
        return Jf

    def loglike_c(self, theta_chol, use_sw=False):
        """
        Parameters
        ----------

        theta_chol: array_like
            The cholesky parameterization of the components

        Returns
        -------
        loglike: scalar
            Log likelihood of the model
        """
        theta = inverse_transform_theta(theta_chol.copy(), self.dims, self.indices)
        theta[-1] = theta_chol[-1]
        return self.loglike(theta, use_sw)

    def gradient_c(self, theta_chol, use_sw=False):
        """
        Parameters
        ----------

        theta_chol: array_like
            The cholesky parameterization of the components

        Returns
        -------
        gradient: array_like
            The gradient of the log likelihood with respect to the covariance
            parameterization

        """
        theta = inverse_transform_theta(theta_chol.copy(), self.dims, self.indices)
        theta[-1] = theta_chol[-1]
        return self.gradient(theta, use_sw)

    def gradient_me_c(self, theta_chol):
        """
        Parameters
        ----------

        theta_chol: array_like
            The cholesky parameterization of the components

        Returns
        -------
        gradient: array_like
            The gradient of the log likelihood with respect to the covariance
            parameterization

        """
        theta = inverse_transform_theta(theta_chol.copy(), self.dims, self.indices)
        theta[-1] = theta_chol[-1]
        return self.gradient_me(theta)

    def hessian_c(self, theta_chol):
        """
        Parameters
        ----------

        theta_chol: array_like
            The cholesky parameterization of the components

        Returns
        -------
        hessian: array_like
            The hessian of the log likelihood with respect to the covariance
            parameterization

        """
        theta = inverse_transform_theta(theta_chol.copy(), self.dims, self.indices)
        theta[-1] = theta_chol[-1]
        return self.hessian(theta)

    def gradient_chol(self, theta_chol, use_sw=False):
        """
        Parameters
        ----------

        theta_chol: array_like
            The cholesky parameterization of the components

        Returns
        -------
        gradient: array_like
            The gradient of the log likelihood with respect to the cholesky
            parameterization

        """
        L_dict = self.update_chol(theta_chol)
        Jf_dict = self.dg_dchol(L_dict)
        Jg = self.gradient_c(theta_chol, use_sw)
        Jf = sp.linalg.block_diag(*Jf_dict.values())
        Jf = np.pad(Jf, [[0, 1]])
        Jf[-1, -1] = 1
        return Jg.dot(Jf)

    def gradient_me_chol(self, theta_chol):
        """
        Parameters
        ----------

        theta_chol: array_like
            The cholesky parameterization of the components

        Returns
        -------
        gradient: array_like
            The gradient of the log likelihood with respect to the cholesky
            parameterization

        """
        L_dict = self.update_chol(theta_chol)
        Jf_dict = self.dg_dchol(L_dict)
        Jg = self.gradient_me_c(theta_chol)
        Jf = sp.linalg.block_diag(*Jf_dict.values())
        Jf = np.pad(Jf, [[0, 1]])
        Jf[-1, -1] = 1
        return Jg.dot(Jf)

    def hessian_chol(self, theta_chol):
        """
        Parameters
        ----------

        theta_chol: array_like
            The cholesky parameterization of the components

        Returns
        -------
        hessian: array_like
            The hessian of the log likelihood with respect to the cholesky
            parameterization

        """
        L_dict = self.update_chol(theta_chol)
        Jf_dict = self.dg_dchol(L_dict)
        Hq = self.hessian_c(theta_chol)
        Jg = self.gradient_c(theta_chol)
        Hf = self.d2g_dchol
        Jf = sp.linalg.block_diag(*Jf_dict.values())
        Jf = np.pad(Jf, [[0, 1]])
        Jf[-1, -1] = 1
        A = Jf.T.dot(Hq).dot(Jf)
        B = np.zeros_like(Hq)

        for key in self.levels:
            ix = self.indices['theta'][key]
            Jg_i = Jg[ix]
            Hf_i = Hf[key]
            C = np.einsum('i,ijk->jk', Jg_i, Hf_i)
            B[ix, ix[:, None]] += C
        H = A + B
        return H

    def _compute_effects(self, theta=None):
        G = self.update_gmat(theta, inverse=False)
        R = self.R * theta[-1]
        V = self.Zs.dot(G).dot(self.Zs.T) + R
        chol_fac = cholesky(V)
        XtVi = (chol_fac.solve_A(self.X)).T
        XtViX = XtVi.dot(self.X)
        XtViX_inv = np.linalg.inv(XtViX)
        beta = _check_shape(XtViX_inv.dot(XtVi.dot(self.y)))
        fixed_resids = _check_shape(self.y) - _check_shape(self.X.dot(beta))
        #Should be G.dot(Z).T.dot(solve(fixed_resids))
        Vinvr = chol_fac.solve_A(fixed_resids)
        u = G.dot(self.Zs.T).dot(Vinvr)

        return beta, XtViX_inv, u, G, R, V

    def _fit(self, use_grad=True, use_hess=False, opt_kws={}):

        if use_grad:
            default_opt_kws = dict(verbose=0, gtol=1e-6, xtol=1e-6)
            if use_hess:
               hess = self.hessian_chol
            else:
                hess = None
            for key, value in default_opt_kws.items():
                if key not in opt_kws.keys():
                    opt_kws[key] = value
            optimizer = sp.optimize.minimize(self.loglike_c, self.theta,
                                             jac=self.gradient_chol, hess=hess,
                                             options=opt_kws, bounds=self.bounds,
                                             method='trust-constr')
        else:
            default_opt_kws = dict(disp=True, gtol=1e-14, ftol=1e-14,
                                   finite_diff_rel_step='3-point', eps=1e-7,
                                   iprint=99)
            for key, value in default_opt_kws.items():
                if key not in opt_kws.keys():
                    opt_kws[key] = value
            optimizer = sp.optimize.minimize(self.loglike_c, self.theta,
                                             bounds=self.bounds_2,
                                             method='L-BFGS-B',
                                             options=opt_kws)
        theta_chol = optimizer.x
        theta = inverse_transform_theta(theta_chol.copy(), self.dims, self.indices)

        beta, XtWX_inv, u, G, R,  V = self._compute_effects(theta)
        params = np.concatenate([beta, theta])
        self.theta, self.beta, self.u, self.params = theta, beta, u, params
        self.Hinv_beta = XtWX_inv
        self.se_beta = np.sqrt(np.diag(XtWX_inv))
        self._G, self._R, self._V = G, R, V
        self.optimizer = optimizer
        self.theta_chol = theta_chol
        self.llconst = (self.X.shape[0] - self.X.shape[1])*np.log(2*np.pi)
        self.lltheta = self.optimizer.fun
        self.ll = (self.llconst + self.lltheta)
        self.llf = self.ll / -2.0

    def _post_fit(self, use_grad=True, analytic_se=False):
        if analytic_se:
            Htheta = self.hessian(self.theta)
        elif use_grad:
            Htheta = so_gc_cd(self.gradient, self.theta)
        else:
            Htheta = so_fc_cd(self.loglike, self.theta)
        self.Hinv_theta = np.linalg.pinv(Htheta/2.0)
        self.se_theta = np.sqrt(np.diag(self.Hinv_theta))
        self.se_params = np.concatenate([self.se_beta, self.se_theta])

    def predict(self, X=None, Z=None):
        if X is None:
            X = self.X
        if Z is None:
            Z = self.Z
        return X.dot(self.beta)+Z.dot(self.u)

    def fit(self, use_grad=True, use_hess=False, analytic_se=False, opt_kws={}):
        self._fit(use_grad, use_hess, opt_kws)
        self._post_fit(use_grad, analytic_se)
        param_names = list(self.fe_vars)
        for level in self.levels:
            for i, j in list(zip(*np.triu_indices(self.dims[level]['n_vars']))):
                param_names.append(f"{level}:G[{i}][{j}]")
        param_names.append("error_cov")
        self.param_names = param_names
        res = np.vstack((self.params, self.se_params)).T
        res = pd.DataFrame(res, index=param_names, columns=['estimate', 'SE'])
        res['t'] = res['estimate'] / res['SE']
        res['p'] = sp.stats.t(self.X.shape[0]-self.X.shape[1]).sf(np.abs(res['t']))
        self.res = res

class WLMEC:

    def __init__(self, formula, data, weights=None, fix_error=False):
        if weights is None:
            weights = np.eye(len(data))
        self.weights = sps.csc_matrix(weights)
        self.weights_inv = sps.csc_matrix(np.linalg.inv(weights))

        indices = {}
        X, Z, y, dims, levels = construct_model_matrices(formula, data)
        theta, theta_indices = make_theta(dims)

        indices['theta'] = theta_indices

        G, g_indices = make_gcov(theta, indices, dims)

        indices['g'] = g_indices

        XZ, Xty, Zty, yty = np.hstack([X, Z]), X.T.dot(y), Z.T.dot(y), y.T.dot(y)

        C, m = sps.csc_matrix(XZ.T.dot(XZ)), sps.csc_matrix(np.vstack([Xty, Zty]))
        M = sps.bmat([[C, m], [m.T, yty]])
        M = M.tocsc()
        self.F = sps.csc_matrix(XZ)
        self.X, self.Z, self.y, self.dims, self.levels = X, Z, y, dims, levels
        self.XZ, self.Xty, self.Zty, self.yty = XZ, Xty, Zty, yty
        self.C, self.m, self.M = C, m, M
        self.theta, self.theta_chol = theta, transform_theta(theta, dims, indices)
        self.G = G
        self.indices = indices
        self.R = sps.eye(Z.shape[0])
        self.Zs = sps.csc_matrix(Z)
        self.jac_mats = get_jacmats2(self.Zs, self.dims, self.indices['theta'],
                                     self.indices['g'], self.theta)
        self.t_indices = list(zip(*np.triu_indices(len(theta))))
        self.elim_mats, self.symm_mats, self.iden_mats = {}, {}, {}
        self.d2g_dchol = {}
        for key in self.levels:
            p = self.dims[key]['n_vars']
            self.elim_mats[key] = lmat(p).A
            self.symm_mats[key] = nmat(p).A
            self.iden_mats[key] = np.eye(p)
            self.d2g_dchol[key] = get_d2_chol(self.dims[key])
        self.bounds = [(0, None) if x==1 else (None, None) for x in self.theta]
        self.fix_error = fix_error

    def update_mme(self, Ginv, Rinv, s):
        C = sps.csc_matrix(self.F.T.dot(Rinv).dot(self.F))
        m = self.F.T.dot(Rinv).dot(self.y)
        yty = np.array(np.atleast_2d(self.y.T.dot(Rinv.A).dot(self.y)))
        M = sps.bmat([[C, m], [m.T, yty]]).tocsc()
        M[-Ginv.shape[0]-1:-1, -Ginv.shape[0]-1:-1] += Ginv
        return M

    def update_gmat(self, theta, inverse=False):
        G = self.G
        for key in self.levels:
            ng = self.dims[key]['n_groups']
            theta_i = theta[self.indices['theta'][key]]
            if inverse:
                theta_i = np.linalg.inv(invech(theta_i)).reshape(-1, order='F')
            else:
                theta_i = invech(theta_i).reshape(-1, order='F')
            G.data[self.indices['g'][key]] = np.tile(theta_i, ng)
        return G

    def loglike(self, theta):
        if self.fix_error:
            s = 1
        else:
            s = theta[-1]
        Ginv = self.update_gmat(theta, inverse=True)
        Rinv = self.weights_inv.dot(self.R /s).dot(self.weights_inv)
        M = self.update_mme(Ginv, Rinv, s)
        logdetG = lndet_gmat(theta, self.dims, self.indices)
        L = np.linalg.cholesky(M.A)
        ytPy = np.diag(L)[-1]**2
        logdetC = np.sum(2*np.log(np.diag(L))[:-1])
        R = (self.weights.dot(self.R * theta[-1]).dot(self.weights))
        logdetR = np.sum(np.log(R.diagonal()))
        ll = logdetR + logdetC + logdetG + ytPy
        return ll

    def gradient(self, theta):
        if self.fix_error:
            s = 1
        else:
            s = theta[-1]
        Ginv = self.update_gmat(theta, inverse=True)
        Rinv = self.weights_inv.dot(self.R / s).dot(self.weights_inv)
        Vinv = sparse_woodbury_inversion(self.Zs, Cinv=Ginv, Ainv=Rinv.tocsc())
        W = (Vinv.dot(self.X))
        XtW = W.T.dot(self.X)
        XtW_inv = np.linalg.inv(XtW)
        P = Vinv - np.linalg.multi_dot([W, XtW_inv, W.T])
        Py = P.dot(self.y)
        grad = []
        for key in (self.levels+['error']):
            for dVdi in self.jac_mats[key]:
                gi = np.einsum("ij,ji->", dVdi.A, P) - Py.T.dot(dVdi.dot(Py))
                grad.append(gi)
        grad = np.concatenate(grad)
        grad = _check_shape(np.array(grad))
        return grad

    def hessian(self, theta):
        if self.fix_error:
            s = 1
        else:
            s = theta[-1]
        Ginv = self.update_gmat(theta, inverse=True)
        Rinv = self.weights_inv.dot(self.R / s).dot(self.weights_inv)
        Vinv = sparse_woodbury_inversion(self.Zs, Cinv=Ginv, Ainv=Rinv.tocsc())
        W = (Vinv.dot(self.X))
        XtW = W.T.dot(self.X)
        XtW_inv = np.linalg.inv(XtW)
        P = Vinv - np.linalg.multi_dot([W, XtW_inv, W.T])
        Py = P.dot(self.y)
        H = []
        PJ, yPJ = [], []
        for key in (self.levels+['error']):
            J_list = self.jac_mats[key]
            for i in range(len(J_list)):
                Ji = J_list[i].T
                PJ.append((Ji.dot(P)).T)
                yPJ.append((Ji.dot(Py)).T)
        t_indices = self.t_indices
        for i, j in t_indices:
            PJi, PJj = PJ[i], PJ[j]
            yPJi, JjPy = yPJ[i], yPJ[j].T
            Hij = -(PJi.dot(PJj)).diagonal().sum()\
                        + (2 * (yPJi.dot(P)).dot(JjPy))[0]
            H.append(np.array(Hij[0]))
        H = invech(np.concatenate(H)[:, 0])
        return H

    def update_chol(self, theta, inverse=False):
        L_dict = {}
        for key in self.levels:
            theta_i = theta[self.indices['theta'][key]]
            L_i = invech_chol(theta_i)
            L_dict[key] = L_i
        return L_dict

    def dg_dchol(self, L_dict):
        Jf = {}
        for key in self.levels:
            L = L_dict[key]
            E = self.elim_mats[key]
            N = self.symm_mats[key]
            I = self.iden_mats[key]
            Jf[key] = E.dot(N.dot(np.kron(L, I))).dot(E.T)
        return Jf

    def loglike_c(self, theta_chol):
        theta = inverse_transform_theta(theta_chol.copy(), self.dims, self.indices)
        theta[-1] = theta_chol[-1]
        return self.loglike(theta)

    def gradient_c(self, theta_chol):
        theta = inverse_transform_theta(theta_chol.copy(), self.dims, self.indices)
        theta[-1] = theta_chol[-1]
        return self.gradient(theta)

    def hessian_c(self, theta_chol):
        theta = inverse_transform_theta(theta_chol.copy(), self.dims, self.indices)
        theta[-1] = theta_chol[-1]
        return self.hessian(theta)

    def gradient_chol(self, theta_chol):
        L_dict = self.update_chol(theta_chol)
        Jf_dict = self.dg_dchol(L_dict)
        Jg = self.gradient_c(theta_chol)
        Jf = sp.linalg.block_diag(*Jf_dict.values())
        Jf = np.pad(Jf, [[0, 1]])
        Jf[-1, -1] = 1
        return Jg.dot(Jf)

    def hessian_chol(self, theta_chol):
        L_dict = self.update_chol(theta_chol)
        Jf_dict = self.dg_dchol(L_dict)
        Hq = self.hessian_c(theta_chol)
        Jg = self.gradient_c(theta_chol)
        Hf = self.d2g_dchol
        Jf = sp.linalg.block_diag(*Jf_dict.values())
        Jf = np.pad(Jf, [[0, 1]])
        Jf[-1, -1] = 1
        A = Jf.T.dot(Hq).dot(Jf)
        B = np.zeros_like(Hq)

        for key in self.levels:
            ix = self.indices['theta'][key]
            Jg_i = Jg[ix]
            Hf_i = Hf[key]
            C = np.einsum('i,ijk->jk', Jg_i, Hf_i)
            B[ix, ix[:, None]] += C
        H = A + B
        return H

    def _compute_effects(self, theta=None):
        G = self.update_gmat(theta, inverse=False)
        Ginv = self.update_gmat(theta, inverse=True)
        R = (self.weights.dot(self.R * theta[-1]).dot(self.weights))
        Rinv = self.weights_inv.dot(self.R / theta[-1]).dot(self.weights_inv)
        V = self.Zs.dot(G).dot(self.Zs.T)+R
        Vinv = sparse_woodbury_inversion(self.Zs, Cinv=Ginv, Ainv=Rinv.tocsc())
        XtVi = (Vinv.dot(self.X)).T
        XtViX = XtVi.dot(self.X)
        XtViX_inv = np.linalg.inv(XtViX)
        beta = _check_shape(XtViX_inv.dot(XtVi.dot(self.y)))
        fixed_resids = _check_shape(self.y) - _check_shape(self.X.dot(beta))

        Zt = self.Zs.T
        u = G.dot(Zt.dot(Vinv)).dot(fixed_resids)

        return beta, XtViX_inv, u, G, R, Rinv, V, Vinv

    def _fit(self, use_hess=False, opt_kws={}):
        default_opt_kws = dict(verbose=3, gtol=1e-6, xtol=1e-6)
        if use_hess:
            hess = self.hessian_chol
        else:
            hess = None
        for key, value in default_opt_kws.items():
            if key not in opt_kws.keys():
                opt_kws[key] = value

        optimizer = sp.optimize.minimize(self.loglike_c, self.theta,
                                              jac=self.gradient_chol, hess=hess,
                                              options=opt_kws, bounds=self.bounds,
                                              method='trust-constr')
        theta_chol = optimizer.x
        theta = inverse_transform_theta(theta_chol, self.dims, self.indices)

        beta, XtWX_inv, u, G, R, Rinv, V, Vinv = self._compute_effects(theta)
        params = np.concatenate([beta, theta])
        self.theta, self.beta, self.u, self.params = theta, beta, u, params
        self.Hinv_beta = XtWX_inv
        self.se_beta = np.sqrt(np.diag(XtWX_inv))
        self._G, self._R, self._Rinv, self._V, self._Vinv = G, R, Rinv, V, Vinv

    def _post_fit(self):
        Htheta = self.hessian(self.theta)
        self.Hinv_theta = np.linalg.pinv(Htheta)
        self.se_theta = np.sqrt(np.diag(self.Hinv_theta))
        self.se_params = np.concatenate([self.se_beta, self.se_theta])

    def predict(self, X=None, Z=None):
        if X is None:
            X = self.X
        if Z is None:
            Z = self.Z
        return X.dot(self.beta)+Z.dot(self.u)

class GLMMC(WLMEC):
    '''
    Currently an ineffecient implementation of a GLMM, mostly done
    for fun.  A variety of implementations for GLMMs have been proposed in the
    literature, and a variety of names have been used to refer to each model;
    the implementation here is based of off linearization using a taylor
    approximation of the error (assumed to be gaussian) around the current
    estimates of fixed and random effects.  This type of approach may be
    referred to as penalized quasi-likelihood, or pseudo-likelihood, and
    may be abbreviated PQL, REPL, RPL, or RQL.

    '''
    def __init__(self, formula, data, weights=None, fam=None):
        if isinstance(fam, ExponentialFamily) is False:
            fam = fam()
        self.f = fam
        self.mod = WLMEC(formula, data, weights=None)
        self.theta_init = self.mod.theta.copy()
        self.mod._fit()
        self.y = self.mod.y
        self.non_continuous = [isinstance(self.f, Binomial),
                               isinstance(self.f, NegativeBinomial),
                               isinstance(self.f, Poisson)]
        if np.any(self.non_continuous):
            self.mod.bounds = self.mod.bounds[:-1]+[(1, 1)]
            self.mod.fix_error=True
        self._nfixed_params = self.mod.X.shape[1]
        self._n_obs = self.mod.X.shape[0]
        self._n_cov_params = len(self.mod.bounds)
        self._df1 = self._n_obs - self._nfixed_params
        self._df2 = self._n_obs - self._nfixed_params - self._n_cov_params - 1
        self._ll_const = self._df1 / 2 * np.log(2*np.pi)

    def _update_model(self, W, nu):
        nu = _check_shape(nu, 2)
        self.mod.weights = sps.csc_matrix(W)
        self.mod.weights_inv = sps.csc_matrix(np.diag(1.0/np.diag((W))))
        self.mod.y = nu
        self.mod.Xty = self.mod.X.T.dot(nu)
        self.mod.Zty = self.mod.Z.T.dot(nu)
        self.mod.theta = self.theta_init
        self.mod.yty = nu.T.dot(nu)

    def _get_pseudovar(self):
        eta = self.mod.predict()
        mu = self.f.inv_link(eta)
        var_mu = _check_shape(self.f.var_func(mu=mu), 1)
        gp = self.f.dlink(mu)
        nu = eta + gp * (_check_shape(self.y, 1) - mu)
        W = np.diag(np.sqrt(var_mu * (self.f.dlink(mu)**2)))
        return W, nu

    def _sandwich_cov(self, r):
        M = self.mod.Hinv_beta
        X = self.mod.X
        Vinv = self.mod.Vinv
        B = (Vinv.dot(X))
        d = _check_shape(r, 2)**2
        B = B * d
        C = B.T.dot(B)
        Cov = M.dot(C).dot(M)
        return Cov

    def predict(self, fixed=True, random=True):
        yhat = 0.0
        if fixed:
            yhat += self.mod.X.dot(self.mod.beta)
        if random:
            yhat += self.mod.Z.dot(self.mod.u)
        return yhat

    def fit(self, n_iters=200, tol=1e-3, optimizer_kwargs={},
            verbose_outer=True, hess=False):
        if 'options' in optimizer_kwargs.keys():
            if 'verbose' not in optimizer_kwargs['options'].keys():
                optimizer_kwargs['options']['verbose'] = 0
        else:
            optimizer_kwargs['options'] = dict(verbose=0)

        if hess:
            hessian = self.mod.hessian
        else:
            hessian = None

        theta = self.mod.theta.copy()
        fit_hist = {}
        for i in range(n_iters):
            W, nu = self._get_pseudovar()
            self._update_model(W, nu)
            self.mod._fit(opt_kws={}, use_hess=hessian)
            tvar = (np.linalg.norm(theta)+np.linalg.norm(self.mod.theta))
            eps = np.linalg.norm(theta - self.mod.theta) / tvar
            fit_hist[i] = dict(param_change=eps, theta=self.mod.theta,
                               nu=nu)
            if verbose_outer:
                print(eps)
            if eps < tol:
                break
            theta = self.mod.theta.copy()
        self.mod._postfit()
        self.res = get_param_table(self.mod.params, self.mod.se_params,
                                   self.mod.X.shape[0]-len(self.mod.params))

        eta_fe = self.predict(fixed=True, random=False)
        eta = self.predict(fixed=True, random=True)
        mu = self.f.inv_link(eta)
        gp = self.f.dlink(mu)
        var_mu  =  _check_shape(self.f.var_func(mu=mu), 1)
        r_eta_fe = _check_shape(self.mod.y, 1) - eta_fe

        generalized_chi2 = r_eta_fe.T.dot(self.mod.Vinv.dot(r_eta_fe))
        resids_raw_linear = _check_shape(self.mod.y, 1) - eta
        resids_raw_mean = _check_shape(self.y, 1) - mu

        var_pearson_linear = self.mod.R.diagonal() / gp**2
        var_pearson_mean = var_mu

        resids_pearson_linear = resids_raw_linear / np.sqrt(var_pearson_linear)
        resids_pearson_mean = resids_raw_mean / np.sqrt(var_pearson_mean)

        pll = self.mod.loglike(self.mod.theta) / -2.0 - self._ll_const
        aicc = -2 * pll + 2 * self._n_cov_params  * self._df1 / self._df2
        bic = -2 * pll + self._n_cov_params * np.log(self._df1)
        self.sumstats = dict(generalized_chi2=generalized_chi2,
                             pseudo_loglike=pll,
                             AICC=aicc,
                             BIC=bic)
        self.resids = dict(resids_raw_linear=resids_raw_linear,
                           resids_raw_mean=resids_raw_mean,
                           resids_pearson_linear=resids_pearson_linear,
                           resids_pearson_mean=resids_pearson_mean)

"""
from pystats.utilities.random_corr import vine_corr
from pystats.tests.test_data import generate_data
from pylmm.pylmm.lmm import LME
from pylmm.pylmm.glmm import WLME, GLMM

from pystats.utilities import numerical_derivs

np.set_printoptions(precision=3, suppress=True, linewidth=200)
formula = "y~1+x1+x2+(1+x3|id1)+(1+x4|id2)"
model_dict = {}
model_dict['gcov'] = {'id1':invech(np.array([2., 0.4, 2.])),
                      'id2':invech(np.array([2.,-0.4, 2.]))}

model_dict['ginfo'] = {'id1':dict(n_grp=200, n_per=10),
                       'id2':dict(n_grp=400, n_per=5)}

model_dict['mu'] = np.zeros(4)
model_dict['vcov'] = vine_corr(4, 20)
model_dict['beta'] = np.array([1, -1, 1])
model_dict['n_obs'] = 2000
data, formula = generate_data(formula, model_dict, r=0.6**0.5)

model_original = LME(formula, data)
model_cholesky = LME3(formula, data)
model_original._fit()
model_cholesky._fit(opt_kws=dict(verbose=3))
model_cholesky._post_fit()

model_original.se_params
model_cholesky.se_params

"""


from flask import render_template, url_for, flash, request
from werkzeug.utils import redirect

from flask_cms.admin.views import AdminView
from flask_cms.admin.widget.forms import WidgetFormFactory, WidgetTypeForm
from flask_cms.ext import db
from flask_cms.utils import get_object_or_404, flash_errors
from flask_cms.widget.models.widget import Widget
from flask_cms.widget.models.widget_type import WidgetType

class CreateWidgetView(AdminView):
    def get(self):
        type = request.args.get('type', None)
        form = WidgetFormFactory(type=type).get_widget_form()
        if type is not None:
            return render_template(
                "widget/{}/create_{}_widget_form.html".format(type, type),
                form=form)
        else:
            return render_template("widget/widget.html", form=form)

    def post(self):
        widget_type_id = int(request.form.get('types'))
        widget_type = get_object_or_404(
            WidgetType, WidgetType.id == widget_type_id)
        form = WidgetFormFactory(
            type=widget_type.name).get_widget_form()

        if form.validate_on_submit():
            widget = Widget(
                name=form.name.data,
                widget_type_id=int(form.types.data)
            )
            db.session.add(widget)
            db.session.commit()
            component = widget.get_component_class()
            component = component.create_by_form(form)
            component.widget = widget
            db.session.add(component)
            db.session.commit()
            flash("Successfully created widget - {}".format(widget.name))
            return redirect(url_for('admin.list_widget'))
        return render_template('widget/widget.html', form=form)

class DeleteWidgetView(AdminView):
    def get(self, widget_id):
        widget = get_object_or_404(Widget, Widget.id == widget_id)
        db.session.delete(widget)
        db.session.commit()
        flash("Successfully deleted widget - {}".format(widget.name))
        return redirect(url_for('admin.list_widget'))

class ShowWidgetView(AdminView):
    def get(self, widget_id):
        widget = get_object_or_404(Widget, Widget.id == widget_id)
        widget_name = widget.widget_type.name
        component = widget.get_component()
        if component is None:
            flash("Widget is empty", "error")
            return redirect(url_for('admin.list_widget'))

        return render_template("widget/{}/show_{}_widget.html".
                               format(widget_name, widget_name),
                               widget=widget,
                               component=component)

class WidgetIndexView(AdminView):
    def get(self):
        widgets = Widget.query.all()
        widget_types = WidgetType.query.all()
        return render_template("widget/list_widget.html",
                               widgets=widgets, widget_types=widget_types)

class CreateWidgetTypeView(AdminView):
    def get(self):
        form = WidgetTypeForm()
        return render_template("widget_type/create_widget_type.html", form=form)

    def post(self):
        form = WidgetTypeForm()
        if form.validate_on_submit():
            widget_type = WidgetType(name=form.name.data)
            db.session.add(widget_type)
            db.session.commit()
            flash("Widget type - {} successfully created".format(widget_type.name))
            return redirect(url_for('admin.list_widget'))

        flash_errors(form)
        return render_template("widget_type/create_widget_type.html", form=form)

class DeleteWidgetTypeView(AdminView):
    def get(self, widget_type_id):
        widget_type = get_object_or_404(WidgetType, WidgetType.id == widget_type_id)
        db.session.delete(widget_type)
        db.session.commit()
        flash("Successfully deleted widget_type - {}".format(widget_type.name))
        return redirect(url_for('admin.list_widget'))


import openai
import time
import os
import ast
import traceback

openai.api_key = os.environ.get('OPENAI_API_KEY')
if not openai.api_key:
    raise ValueError("Please set the OPENAI_API_KEY environment variable.")

def send_openai_request(prompt, engine = "gpt-3.5-turbo"):
    try:
        response = openai.ChatCompletion.create(
            model=engine,
            messages=[{'role': 'user', 'content': prompt}],
            temperature=0,
            stream=True
        )

        collected_messages = []
        for chunk in response:
            chunk_message = str(chunk['choices'][0]['delta']['content'])
            print(chunk_message)
            collected_messages.append(chunk_message)

        return collected_messages

    except Exception as e:
        print(f"Got exception when calling openai {e}")
        traceback.print_exc()  # This will print the full traceback
        return "Error calling model"

prompt = "count to 10"
print(send_openai_request(prompt))


from django import template
from django.utils.safestring import mark_safe
import bleach

register = template.Library()

_ALLOWED_ATTRIBUTES = {
        'a': ['href', 'title'],
        'table': ['class']
}
_ALLOWED_TAGS = ['b', 'i', 'ul', 'li', 'p', 'br', 'a', 'pre','code','h1', 'h2', 'h3', 'h4', 'ol','strong', 'em', 'blockquote',
    'table', 'thead', 'tr', 'td', 'tbody', 'th']

@register.filter()
def xsafe(text):
    return mark_safe(bleach.clean(text, tags=_ALLOWED_TAGS, attributes=_ALLOWED_ATTRIBUTES))