#!/usr/bin/env python3

from lxml import etree as ET
import base64
from . computations import Computations
import sys

nsmap = {
    "http://www.hmrc.gov.uk/schemas/ct/comp/2023-01-01": "ct-comp",
    "http://www.hmrc.gov.uk/schemas/ct/dpl/2021-01-01": "dpl",
    "http://xbrl.frc.org.uk/fr/2023-01-01/core": "uk-core"
}

ct_ns = "http://www.govtalk.gov.uk/taxation/CT/5"

class Box:
    def __init__(self, id, kind=None):
        self.kind = kind
        self.id = id

    def present(self, obj):
        if self.id in obj.form_values["ct600"]:

            value = obj.form_values["ct600"][self.id]
            if value == None:
                return False

            return True

        return False

    def fetch(self, obj):
        return obj.form_values["ct600"][self.id]

    def get(self, obj):
        value = self.fetch(obj)

        if self.kind == "yesno":
            return "yes" if value else "no"

        if self.kind == "money":
            return "%.2f" % value

        if self.kind == "rate":
            return "%.2f" % value

        if self.kind == "pounds":
            return "%.2f" % int(value)

        if self.kind == "yes":
            return "yes" if value else "FIXME"

        if self.kind == "date":
            return str(value)

        if self.kind == "year":
            return str(value)

        if self.kind == "companytype":
            return "%02d" % value

        return value

class Fixed(Box):
    def __init__(self, value, kind=None):
        self.value = value
        self.kind = kind

    def present(self, obj):
        return True

    def fetch(self, obj):
        return self.value

class InputBundle:

    def __init__(self, comps, accts, form_values, params, atts):
        self.comps = comps
        self.accts = accts
        self.form_values = form_values
        self.params = params
        self.atts = atts

    def box(self, num):
        val = self.form_values["ct600"][num]
        if val is None: return ""
        return val

    def date(self, num):
        val = self.form_values["ct600"][num]
        if val is None: return ""
        return str(val)

    def irheader(self):

        irheader = ET.Element(
            "{%s}%s" % (ct_ns, "IRheader"),
            nsmap={"ct": ct_ns}
        )

        keys = ET.SubElement(irheader, "{%s}%s" % (ct_ns, "Keys"))

        ET.SubElement(
            keys, "{%s}%s" % (ct_ns, "Key"), Type="UTR"
        ).text = str(self.box(3))

        ET.SubElement(
            irheader, "{%s}%s" % (ct_ns, "PeriodEnd")
        ).text = str(self.date(35))

        principal = ET.SubElement(irheader, "{%s}%s" % (ct_ns, "Principal"))
        contact = ET.SubElement(principal, "{%s}%s" % (ct_ns, "Contact"))
        name = ET.SubElement(contact, "{%s}%s" % (ct_ns, "Name"))
        ET.SubElement(
            name, "{%s}%s" % (ct_ns, "Ttl")
        ).text = self.params.get("title")
        ET.SubElement(
            name, "{%s}%s" % (ct_ns, "Fore")
        ).text = self.params.get("first-name")
        ET.SubElement(
            name, "{%s}%s" % (ct_ns, "Sur")
        ).text = self.params.get("second-name")
        ET.SubElement(
            contact, "{%s}%s" % (ct_ns, "Email")
        ).text = self.params.get("email")
        tel = ET.SubElement(contact, "{%s}%s" % (ct_ns, "Telephone"))
        ET.SubElement(
            tel, "{%s}%s" % (ct_ns, "Number")
        ).text = self.params.get("phone")

        ET.SubElement(irheader, "{%s}%s" % (ct_ns, "IRmark")).text = ""
        ET.SubElement(irheader, "{%s}%s" % (ct_ns, "Sender")).text = "Company"

        return irheader

    def get_return(self):

        comp_ixbrl = base64.b64encode(self.comps).decode("utf-8")
        accounts_ixbrl = base64.b64encode(self.accts).decode("utf-8")

        mapping = {
            "CompanyInformation": {
                "CompanyName": Box(1),
                "RegistrationNumber": Box(2),
                "Reference": Box(3),
                "CompanyType": Box(4, kind="companytype"),
                "NorthernIreland": {
                    "NItradingActivity": Box(5, kind="yesno"),
                    "SME": Box(6, kind="yesno"),
                    "NIemployer": Box(7, kind="yesno"),
                    "SpecialCircumstances": Box(8, kind="yesno"),
                },
                "PeriodCovered": {
                    "From": Box(30, kind="date"),
                    "To": Box(35, kind="date"),
                }
            },
            "ReturnInfoSummary": {
                "ThisPeriod": Box(40, kind="yesno"),
                "EarlierPeriod": Box(45, kind="yesno"),
                "MultipleReturns": Box(50, kind="yesno"),
                "ProvisionalFigures": Box(55, kind="yesno"),
                "PartOfNonSmallGroup": Box(60, kind="yesno"),
                "RegisteredAvoidanceScheme": Box(65, kind="yesno"),
                "TransferPricing": {
                    "Adjustment": Box(70, kind="yesno"),
                    "SME": Box(75, kind="yesno"),
                },
                "Accounts": {
                    "ThisPeriodAccounts": Box(80, kind="yesno"),
                    "DifferentPeriod": Box(85, kind="yesno"),
                    "NoAccountsReason": Box(90),
                },
                "Computations": {
                    "ThisPeriodComputations": Box(80, kind="yesno"),
                    "DifferentPeriod": Box(85, kind="yesno"),
                    "NoComputationsReason": Box(90),
                },
                "SupplementaryPages": {
                    "CT600A": Box(95, kind="yesno"),
                    "CT600B": Box(100, kind="yesno"),
                    "CT600C": Box(105, kind="yesno"),
                    "CT600D": Box(110, kind="yesno"),
                    "CT600E": Box(115, kind="yesno"),
                    "CT600F": Box(120, kind="yesno"),
                    "CT600G": Box(125, kind="yesno"),
                    "CT600H": Box(130, kind="yesno"),
                    "CT600I": Box(135, kind="yesno"),
                    "CT600J": Box(140, kind="yesno"),
                    "CT600K": Box(141, kind="yesno"),
                    "CT600L": Box(142, kind="yesno"),
                    "CT600N": Box(144, kind="yesno"),
                }
            },
            "Turnover": {
                "Total": Box(145, kind="pounds"),
                "OtherFinancialConcerns": Box(150, kind="pounds"),
            },
            "CompanyTaxCalculation": {
                "Income": {
                    "Trading": {
                        "Profits": Box(155, kind="pounds"),
                        "LossesBroughtForward": Box(160, kind="pounds"),
                        "NetProfits": Box(165, kind="pounds"),
                    },
                    "NonTradingLoanProfitsAndGains": Box(170, kind="pounds"),
                    "IncomeStatedNet": Box(172, kind="yesno"),
                    "NonLoanAnnuitiesAnnualPaymentsDiscounts": Box(175, kind="pounds"),
                    "NonUKdividends": Box(180, kind="pounds"),
                    "DeductedIncome": Box(185, kind="pounds"),
                    "PropertyBusinessIncome": Box(190, kind="pounds"),
                    "NonTradingGainsIntangibles": Box(195, kind="pounds"),
                    "TonnageTaxProfits": Box(200, kind="pounds"),
                    "OtherIncome": Box(205, kind="pounds"),
                },
                "ChargeableGains": {
                    "GrossGains": Box(210, kind="pounds"),
                    "AllowableLosses": Box(215, kind="pounds"),
                    "NetChargeableGains": Box(220, kind="pounds"),
                },
                "LossesBroughtForward": Box(225, kind="pounds"),
                "NonTradeDeficitsOnLoans": Box(230, kind="pounds"),
                "ProfitsBeforeOtherDeductions": Box(235, kind="pounds"),
                "DeductionsAndReliefs": {
                    "UnquotedShares": Box(240, kind="pounds"),
                    "ManagementExpenses": Box(245, kind="pounds"),
                    "UKpropertyBusinessLosses": Box(250, kind="pounds"),
                    "CapitalAllowances": Box(255, kind="pounds"),
                    "NonTradeDeficits": Box(260, kind="pounds"),
                    "CarriedForwardNonTradeDeficits": Box(263, kind="pounds"),
                    "NonTradingLossIntangibles": Box(265, kind="pounds"),
                    "TradingLosses": Box(275, kind="pounds"),
                    "TradingLossesCarriedBack": Box(280, kind="yesno"),
                    "TradingLossesCarriedForward": Box(285, kind="pounds"),
                    "NonTradeCapitalAllowances": Box(290, kind="pounds"),
                    "Total": Box(295, kind="pounds"),
                },
                "ChargesAndReliefs": {
                    "ProfitsBeforeDonationsAndGroupRelief": Box(
                        300, kind="pounds"
                    ),
                    "QualifyingDonations": Box(305, kind="pounds"),
                    "GroupRelief": Box(310, kind="pounds"),
                    "GroupReliefForCarriedForwardLosses": Box(312, kind="pounds"),
                },
                "ChargeableProfits": Box(315, kind="pounds"),
                "RingFenceProfitsIncluded": Box(320, kind="pounds"),
                "NorthernIrelandProfitsIncluded": Box(325, kind="pounds"),
                "CorporationTaxChargeable": {
                    "AssociatedCompanies": {
                        "ThisPeriod": Box(326),
                        "AssociatedCompaniesFinancialYears": {
                            "FirstYear": Box(327),
                            "SecondYear": Box(328)
                        },
                        "StartingOrSmallCompaniesRate": Box(329, kind="yesno")
                    },
                    "FinancialYearOne": {
                        "Year": Box(330, kind="year"),
                        "Details": {
                            "Profit": Box(335, kind="pounds"),
                            "TaxRate": Box(340, kind="rate"),
                            "Tax": Box(345, kind="money")
                        }
                    },
                    "FinancialYearTwo": {
                        "Year": Box(380, kind="year"),
                        "Details": {
                            "Profit": Box(385, kind="pounds"),
                            "TaxRate": Box(390, kind="rate"),
                            "Tax": Box(395, kind="money")
                        }
                    },
                },
                "CorporationTax": Box(430, kind="money"),
                "MarginalReliefForRingFenceTrades": Box(435, kind="money"),
                "NetCorporationTaxChargeable": Box(440, kind="money"),
                "TaxReliefsAndDeductions": {
                    "CommunityInvestmentRelief": Box(445, kind="money"),
                    "DoubleTaxation": {
                        "DoubleTaxationRelief": Box(450, kind="money"),
                        "UnderlyingRate": Box(455, kind="yesno"),
                        "AmountCarriedBack": Box(460, kind="yes"),
                    },
                    "AdvancedCorporationTax": Box(465, kind="money"),
                    "TotalReliefsAndDeductions": Box(470, kind="money"),
                },
                "CJRS": {
                    "CJRSreceived": Box(471, kind="money"),
                    "CJRSdue": Box(472, kind="money"),
                    "CJRSoverpaymentAlreadyAssessed": Box(473, kind="money"),
                    "JobRetentionBonusOverpayment": Box(474, kind="money"),
                },
            },
            "CalculationOfTaxOutstandingOrOverpaid": {
                "NetCorporationTaxLiability": Box(475, kind="money"),
                "LoansToParticipators": Box(480, kind="money"),
                "CT600AreliefDue": Box(485, kind="yesno"),
                "CFCtaxPayable": Box(490, kind="money"),
                "BankLevyPayable": Box(495, kind="money"),
                "BankSurchargePayable": Box(496, kind="money"),
                "RPDTpayable": Box(497, kind="money"),
                "CFCandBankLevyTotal": Box(500, kind="money"),
                "EOGPLpayable": Box(501, kind="money"),
                "SupplementaryCharge": Box(505, kind="money"),
                "TaxChargeable": Box(510, kind="money"),
                "IncomeTax": {
                    "DeductedIncomeTax": Box(515, kind="money"),
                    "TaxRepayable": Box(520, kind="money"),
                },
                "TaxPayable": Box(525, kind="money"),
                "CJRSoverpaymentsNowDue": Box(526, kind="money"),
                "RestitutionTax": Box(527, kind="money"),
                "TaxPayableIncludingRestitutionTax": Box(528, kind="money"),
            },
            "TaxReconciliation": {
                "ResearchAndDevelopmentCredit": Box(530, kind="money"),
                "VaccineCredit": Box(535, kind="money"),
                "CreativeCredit": Box(540, kind="money"),
                "ResearchAndDevelopmentVaccineOrCreativeTaxCredit":  Box(545, kind="money"),
                "LandRemediationCredit": Box(550, kind="money"),
                "LifeAssuranceCompanyCredit": Box(555, kind="money"),
                "LandOrLifeCredit": Box(560, kind="money"),
                "CapitalAllowancesFirstYearCredit": Box(565, kind="money"),
                "SurplusResearchAndDevelopmentCreditsOrCreativeCreditPayable": Box(570, kind="money"),
                "LandOrLifeCreditPayable": Box(575, kind="money"),
                "CapitalAllowancesFirstYearCreditPayable": Box(580, kind="money"),
                "RingFenceCorpTaxIncluded": Box(585, kind="money"),
                "NIcorporationTaxIncluded": Box(586, kind="money"),
                "RingFenceSupplementaryChargeIncluded": Box(590, kind="money"),
                "TaxAlreadyPaid": Box(595, kind="money"),
                "TaxOutstandingOrOverpaid": {
                    "TaxOutstanding": Box(600, kind="money"),
                    "TaxOverpaid": Box(605, kind="money"),
                },
                "RefundsSurrendered": Box(610, kind="money"),
                "RandDExpenditureCreditsSurrendered": Box(615, kind="money"),
            },
            "IndicatorsAndInformation": {
                "FrankedInvestmentIncome": Box(620, kind="pounds"),
                "NumberOf51groupCompanies": Box(625),
                "InstalmentPayments": Box(630, kind="yes"),
                "VeryLargeQIPs": Box(631, kind="yes"),
                "GroupPayment": Box(635, kind="yes"),
                "IntangibleAssets": Box(640, kind="yes"),
                "CrossBorderRoyalty": Box(645, kind="yes"),
                "EatOutToHelpOutScheme": Box(647, kind="pounds"),
            },
            "EnhancedExpenditure": {
                "SMEclaim": Box(650, kind="yes"),
                "LargeCompanyClaim": Box(655, kind="yes"),
                "RAndDClaimNotificationForm": Box(656, kind="yesno"),
                "AdditionalRAndDForm": Box(657, kind="yesno"),
                "RAndDExpenditureSME": Box(659, kind="pounds"),
                "RandDEnhancedExpenditure": Box(660, kind="pounds"),
                "CreativeEnhancedExpenditure": Box(665, kind="pounds"),
                "RandDAndCreativeEnhancedExpenditure": Box(670, kind="pounds"),
                "SMEclaimAsLargeCompany": Box(675, kind="pounds"),
                "VaccineResearch": Box(680, kind="pounds"),
            },
            "LandRemediationEnhancedExpenditure": Box(685, kind="pounds"),
            "AllowancesAndCharges": {
                "AIACapitalAllowancesInc": Box(690, kind="pounds"),
                "MachineryAndPlantSpecialRatePool": {
                    "BalancingCharges": Box(695, kind="pounds"),
                    "CapitalAllowances": Box(700, kind="pounds"),
                },
                "MachineryAndPlantMainPool": {
                    "BalancingCharges": Box(705, kind="pounds"),
                    "CapitalAllowances": Box(710, kind="pounds"),
                },
                "StructuresAndBuildingsCapitalAllowances": Box(711, kind="pounds"),
                "ElectricChargePoints": {
                    "BalancingCharges": Box(713, kind="pounds"),
                    "CapitalAllowances": Box(714, kind="pounds"),
                },
                "BusinessPremisesRenovationIncluded": {
                    "BalancingCharges": Box(715, kind="pounds"),
                    "CapitalAllowances": Box(720, kind="pounds"),
                },
                "EnterpriseZones": {
                    "BalancingCharges": Box(721, kind="pounds"),
                    "CapitalAllowances": Box(722, kind="pounds"),
                },
                "ZeroEmissionsGoodsVehicles": {
                    "BalancingCharges": Box(723, kind="pounds"),
                    "CapitalAllowances": Box(724, kind="pounds"),
                },
                "ZeroEmissionsCars": {
                    "BalancingCharges": Box(726, kind="pounds"),
                    "CapitalAllowances": Box(727, kind="pounds"),
                },
                "Others": {
                    "BalancingCharges": Box(725, kind="pounds"),
                    "CapitalAllowances": Box(730, kind="pounds"),
                },
            },
            "NotIncluded": {
                "AIACapitalAllowancesNotInc": Box(735, kind="pounds"),
                "StructuresAndBuildingsCapitalAllowances": Box(736, kind="pounds"),
                "ElectricChargePoints": {
                    "BalancingCharges": Box(737, kind="pounds"),
                    "CapitalAllowances": Box(738, kind="pounds"),
                },
                "BusinessPremisesRenovationNotIncluded": {
                    "BalancingCharges": Box(740, kind="pounds"),
                    "CapitalAllowances": Box(745, kind="pounds"),
                },
                "EnterpriseZones": {
                    "BalancingCharges": Box(746, kind="pounds"),
                    "CapitalAllowances": Box(747, kind="pounds"),
                },
                "ZeroEmissionsGoodsVehicles": {
                    "BalancingCharges": Box(748, kind="pounds"),
                    "CapitalAllowances": Box(749, kind="pounds"),
                },
                "ZeroEmissionsCars": {
                    "BalancingCharges": Box(751, kind="pounds"),
                    "CapitalAllowances": Box(752, kind="pounds"),
                },
                "OtherAllowancesAndCharges": {
                    "BalancingCharges": Box(750, kind="pounds"),
                    "CapitalAllowances": Box(755, kind="pounds"),
                },
            },
            "QualifyingExpenditure": {
                "MachineryAndPlantExpenditure": Box(760, kind="pounds"),
                "DesignatedEnvironmentallyFriendlyMachineryAndPlant": Box(765, kind="pounds"),
                "MachineryAndPlantLongLife": Box(770, kind="pounds"),
                "StructuresAndBuildings": Box(771, kind="pounds"),
                "OtherMachineryAndPlant": Box(775, kind="pounds"),
            },
            "LossesDeficitsAndExcess": {
                "AmountArising": {
                    "LossesOfTradesUK": {
                        "Arising": Box(780, kind="pounds"),
                        "SurrenderMaximum": Box(785, kind="pounds"),
                    },
                    "LossesOfTradesOutsideUK": Box(790, kind="pounds"),
                    "Loans": {
                        "Arising": Box(795, kind="pounds"),
                        "SurrenderMaximum": Box(800, kind="pounds"),
                    },
                    "UKpropertyBusinessLosses": {
                        "Arising": Box(805, kind="pounds"),
                        "SurrenderMaximum": Box(810, kind="pounds"),
                    },
                    "OverseasPropertyBusinessLosses": Box(815, kind="pounds"),
                    "MiscLosses": Box(820, kind="pounds"),
                    "CapitalLosses": Box(825, kind="pounds"),
                    "NonTradingLossesIntangibles": {
                        "Arising": Box(830, kind="pounds"),
                        "SurrenderMaximum": Box(835, kind="pounds"),
                    },
                },
                "ExcessAmounts": {
                    "NonTradeCapital": Box(840, kind="pounds"),
                    "QualifyingDonations": Box(845, kind="pounds"),
                    "ManagementExpenses": {
                        "Arising": Box(850, kind="pounds"),
                        "SurrenderMaximum": Box(855, kind="pounds"),
                    },
                }
            },
            "NorthernIrelandInformation": {
                "NIagainstUK": Box(856, kind="pounds"),
                "NIagainstNI": Box(857, kind="pounds"),
                "UKagainstNI": Box(858, kind="pounds"),
            },
            "OverpaymentsAndRepayments": {
                "OwnRepaymentsLowerLimit": Box(860, kind="pounds"),
                "RepaymentsForThePeriodCoveredByThisReturn": {
                    "CorporationTax": Box(865, kind="money"),
                    "IncomeTax": Box(870, kind="money"),
                    "RandDTaxCredit": Box(875, kind="money"),
                    "RandDExpenditureCredit": Box(880, kind="money"),
                    "CreativeCredit": Box(885, kind="money"),
                    "LandRemediationCredit": Box(890, kind="money"),
                    "PayableCapitalAllowancesFirstYearCredit": Box(895, kind="money"),
                },
                "Surrender": {
                    "Amount": Box(900, kind="money"),
                    "JointNotice": {
                        "Attached": Box(905, kind="yes"),
                        "WillFollow": Box(910, kind="yes"),
                    },
                    "StopUntilNotice": Box(915, kind="money"),
                },
                "BankAccountDetails": {
                    "BankName": Box(920),
                    "SortCode": Box(925),
                    "AccountNumber": Box(930),
                    "AccountName": Box(935),
                    "BuildingSocReference": Box(940),

                },
                "PaymentToPerson": {
                    "Recipient": Box(955),
                    "Address": {

                        # FIXME
                        "Line": Box(960),

                        # FIXME: AdditionalLine
                        # FIXME: PostCode

                    },
                    "NomineeReference": Box(965),
                },
            },
            "Declaration": {
                "AcceptDeclaration": Fixed("yes"),
                "Name": Box(975),
                "Status": Box(985),
            },
            "AttachedFiles": {
                "XBRLsubmission": {
                    "Computation": {
                        "Instance": {
                            "EncodedInlineXBRLDocument": Fixed(comp_ixbrl),
                        }
                    },
                    "Accounts": {
                        "Instance": {
                            "EncodedInlineXBRLDocument": Fixed(accounts_ixbrl),
                        }
                    }
                }
            },
            "EnergyProfitsLevy": {
                "EOGPLamounts": Box(986, kind="pounds"),
            }
        }

        mapping2 = {
            "CompanyInformation": {
                "CompanyName": Box(1),
                "RegistrationNumber": Box(2),
                "Reference": Box(3),
                "CompanyType": Box(4, kind="companytype"),
                "NorthernIreland": {
                    "NItradingActivity": Box(5, kind="yesno"),
                    "SME": Box(6, kind="yesno"),
                    "NIemployer": Box(7, kind="yesno"),
                    "SpecialCircumstances": Box(8, kind="yesno"),
                },
                "PeriodCovered": {
                    "From": Box(30, kind="date"),
                    "To": Box(35, kind="date"),
                }
            }
        }

        ctr = ET.Element(
            "{%s}%s" % (ct_ns, "CompanyTaxReturn"),
            nsmap={"ct": ct_ns}
        )

        def add_to_tree(root, tree):

            for key, value in tree.items():

                if type(value) == dict:

                    tag = "{%s}%s" % (ct_ns, key)
                    elt = ET.Element(tag)
                    add_to_tree(elt, value)
                    if len(elt) > 0:
                        root.append(elt)

                elif isinstance(value, Box):

                    if value.present(self):

                        tag = "{%s}%s" % (ct_ns, key)

                        eltval = value.get(self)

                        if isinstance(eltval, list):
                            for it in eltval:
                                itval = str(it)
                                ET.SubElement(root, tag).text = itval

                        else:
                            eltval = str(eltval)
                            ET.SubElement(root, tag).text = eltval

        add_to_tree(ctr, mapping)

        irenv = ET.Element(
            "{%s}%s" % (ct_ns, "IRenvelope"), nsmap={"ct": ct_ns}
        )

        irenv.append(self.irheader())
        irenv.append(ctr)

        # CompanyTaxReturn needs an attribute
        irenv.find(".//{%s}CompanyTaxReturn[1]" % ct_ns).set(
            "ReturnType", "new"
        )

        attchs = irenv.find(".//{%s}AttachedFiles[1]" % ct_ns)

        for f in self.atts:

            raw = self.atts[f]
            body = base64.b64encode(raw).decode("utf-8")

            att = ET.Element(
                "{%s}%s" % (ct_ns, "Attachment"),
                nsmap={"ct": ct_ns}
            )

            att.text = body

            att.set("Filename", f)
            att.set("Description", "supporting document")
            att.set("Format", "pdf")
            att.set("Size", str(len(raw)))
            att.set("Type", "other")

            attchs.append(att)

        return ET.ElementTree(irenv)


import time
import sys

from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException, \
    WebDriverException, TimeoutException, NoSuchWindowException, \
    StaleElementReferenceException, ElementNotInteractableException
from selenium.webdriver import ActionChains
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from regression.feature_utils.locators import QueryToolLocators, \
    NavMenuLocators, ConnectToServerDiv
from regression.feature_utils.tree_area_locators import TreeAreaLocators

class PgadminPage:
    """
    Helper class for interacting with the page, given a selenium driver
    """
    # Common argument passed for scrolling
    js_executor_scrollintoview_arg = "arguments[0].scrollIntoView()"

    def __init__(self, driver, app_config):
        self.driver = driver
        self.app_config = app_config
        self.timeout = 30
        self.app_start_timeout = 90

    def reset_layout(self):
        attempt = 0
        while attempt < 4:
            try:
                self.click_element(self.find_by_partial_link_text("File"))
                break
            except (TimeoutException, NoSuchWindowException):
                self.driver.refresh()
                try:
                    WebDriverWait(self.driver, 3).until(EC.alert_is_present())
                    self.driver.switch_to_alert().accept()
                    attempt = attempt + 1
                except TimeoutException:
                    attempt = attempt + 1

        self.find_by_partial_link_text("Reset Layout").click()
        self.click_modal('OK')
        self.wait_for_reloading_indicator_to_disappear()

    def refresh_page(self):
        self.driver.refresh()

    def click_modal(self, button_text):
        time.sleep(0.5)
        # Find active alertify dialog in case of multiple alertify dialog
        # & click on that dialog
        modal_button = self.find_by_xpath(
            "//div[contains(@class, 'alertify') and "
            "not(contains(@class, 'ajs-hidden'))]//button[.='%s']"
            % button_text)
        self.click_element(modal_button)

    def add_server(self, server_config):
        self.find_by_xpath(
            "//*[@class='aciTreeText' and contains(.,'Servers')]").click()

        if self.driver.name == 'firefox':
            ActionChains(self.driver).context_click(self.find_by_xpath(
                "//*[@class='aciTreeText' and contains(.,'Servers')]"))\
                .perform()
            ActionChains(self.driver).move_to_element(
                self.find_by_xpath("//li/span[text()='Create']")).perform()
            self.find_by_xpath("//li/span[text()='Server...']").click()
        else:
            self.driver.find_element_by_link_text("Object").click()
            ActionChains(self.driver).move_to_element(
                self.driver.find_element_by_link_text("Create")).perform()
            self.find_by_partial_link_text("Server...").click()

        WebDriverWait(self.driver, 5).until(EC.visibility_of_element_located(
            (By.XPATH, "//div[text()='Create - Server']")))

        # After server dialogue opens
        self.fill_input_by_field_name("name", server_config['name'],
                                      loose_focus=True)
        self.find_by_partial_link_text("Connection").click()
        self.fill_input_by_field_name("host", server_config['host'])
        self.fill_input_by_field_name("port", server_config['port'])
        self.fill_input_by_field_name("username", server_config['username'])
        self.fill_input_by_field_name("password", server_config['db_password'])
        WebDriverWait(self.driver, 10).until(EC.element_to_be_clickable(
            (By.CSS_SELECTOR, "button[type='save'].btn.btn-primary")))
        self.find_by_css_selector("button[type='save'].btn.btn-primary").\
            click()

        server_tree_xpath = \
            "//*[@id='tree']//*[.='" + server_config['name'] + "']"
        try:
            WebDriverWait(self.driver, 10).until(
                EC.visibility_of_element_located(
                    (By.XPATH, server_tree_xpath)))
        except TimeoutException:
            self.toggle_open_servers_group()
            WebDriverWait(self.driver, 10).until(
                EC.visibility_of_element_located(
                    (By.XPATH, server_tree_xpath)))

    def open_query_tool(self):
        self.driver.find_element_by_link_text("Tools").click()
        tools_menu = self.driver.find_element_by_id('mnu_tools')

        query_tool = tools_menu.find_element_by_id('query_tool')

        self.enable_menu_item(query_tool, 10)

        self.find_by_partial_link_text("Query Tool").click()

    def open_view_data(self, table_name):
        self.driver.find_element_by_link_text("Object").click()
        ActionChains(
            self.driver
        ).move_to_element(
            self.driver.find_element_by_link_text("View/Edit Data")
        ).perform()
        self.find_by_partial_link_text("All Rows").click()
        time.sleep(1)
        # wait until datagrid frame is loaded.

        self.click_tab(table_name)

        WebDriverWait(self.driver, self.timeout).until(
            EC.visibility_of_element_located(
                (By.CSS_SELECTOR, 'iframe')
            ), "Timed out waiting for div element to appear"
        )
        self.driver.switch_to.frame(
            self.driver.find_element_by_tag_name('iframe')
        )

    def enable_menu_item(self, menu_item, wait_time):
        start_time = time.time()
        # wait until menu becomes enabled.
        while time.time() - start_time < wait_time:  # wait_time seconds
            # if menu is disabled then it will have
            # two classes 'dropdown-item disabled'.
            # And if menu is enabled the it will have
            # only one class 'dropdown-item'.

            if 'dropdown-item' == str(menu_item.get_attribute('class')):
                break
            time.sleep(0.1)
        else:
            assert False, "'Tools -> Query Tool' menu did not enable."

    def close_query_tool(self, prompt=True):
        self.driver.switch_to.default_content()
        tab = self.find_by_xpath(
            "//div[@class='wcPanelTab wcPanelTabActive']")
        ActionChains(self.driver).context_click(tab).perform()
        self.find_by_xpath(
            "//li[contains(@class, 'context-menu-item')]/span[contains(text(),"
            " 'Remove Panel')]").click()
        if prompt:
            self.driver.switch_to.frame(
                self.driver.find_elements_by_tag_name("iframe")[0])
            time.sleep(.5)
            self.click_element(self.find_by_xpath(
                '//button[contains(@class, "ajs-button") and '
                'contains(.,"Don\'t save")]'))

            if self.check_if_element_exist_by_xpath(
                    "//button[text()='Rollback']", 1):
                self.click_element(
                    self.find_by_xpath("//button[text()='Rollback']"))
        self.driver.switch_to.default_content()

    def clear_query_tool(self):
        self.click_element(
            self.find_by_css_selector(QueryToolLocators.btn_clear_dropdown)
        )
        ActionChains(self.driver).move_to_element(
            self.find_by_css_selector(QueryToolLocators.btn_clear)).perform()
        self.click_element(
            self.find_by_css_selector(QueryToolLocators.btn_clear)
        )
        self.click_modal('Yes')

    def execute_query(self, query):
        self.fill_codemirror_area_with(query)
        self.click_execute_query_button()

    def click_execute_query_button(self, timeout=20):
        retry = 5
        execute_button = self.find_by_css_selector(
            QueryToolLocators.btn_execute_query_css)
        first_click = execute_button.get_attribute('data-click-counter')
        while retry > 0:
            execute_button.click()
            execute_button = self.find_by_css_selector(
                QueryToolLocators.btn_execute_query_css)
            second_click = execute_button.get_attribute(
                'data-click-counter')
            if first_click != second_click:
                self.wait_for_query_tool_loading_indicator_to_appear()
                break
            else:
                retry -= 1
        self.wait_for_query_tool_loading_indicator_to_disappear(timeout)

    def check_execute_option(self, option):
        """"This function will check auto commit or auto roll back based on
        user input. If button is already checked, no action will be taken"""
        query_options = self.driver.find_element_by_css_selector(
            QueryToolLocators.btn_query_dropdown)
        expanded = query_options.get_attribute("aria-expanded")
        if expanded == "false":
            query_options.click()

        def update_execute_option_setting(
                css_selector_of_option_status, css_selector_of_option,):
            retry = 3
            check_status = self.driver.find_element_by_css_selector(
                css_selector_of_option_status)
            if 'visibility-hidden' in check_status.get_attribute('class'):
                while retry > 0:
                    self.find_by_css_selector(css_selector_of_option).click()
                    time.sleep(0.2)
                    if 'visibility-hidden' not in \
                            check_status.get_attribute('class'):
                        break
                    else:
                        retry -= 1
        if option == 'auto_commit':
            update_execute_option_setting(
                QueryToolLocators.btn_auto_commit_check_status,
                QueryToolLocators.btn_auto_commit)
        if option == 'auto_rollback':
            update_execute_option_setting(
                QueryToolLocators.btn_auto_rollback_check_status,
                QueryToolLocators.btn_auto_rollback)

    def uncheck_execute_option(self, option):
        """"This function will uncheck auto commit or auto roll back based on
        user input. If button is already unchecked, no action will be taken"""
        query_options = self.driver.find_element_by_css_selector(
            QueryToolLocators.btn_query_dropdown)
        expanded = query_options.get_attribute("aria-expanded")
        if expanded == "false":
            query_options.click()

        def update_execute_option_setting(
                css_selector_of_option_status, css_selector_of_option):
            retry = 3
            check_status = self.driver.find_element_by_css_selector(
                css_selector_of_option_status)
            if 'visibility-hidden' not in check_status.get_attribute('class'):
                while retry > 0:
                    self.find_by_css_selector(
                        css_selector_of_option).click()
                    time.sleep(0.2)
                    if 'visibility-hidden' in \
                            check_status.get_attribute('class'):
                        break
                    else:
                        retry -= 1

        if option == 'auto_commit':
            update_execute_option_setting(
                QueryToolLocators.btn_auto_commit_check_status,
                QueryToolLocators.btn_auto_commit)
        if option == 'auto_rollback':
            update_execute_option_setting(
                QueryToolLocators.btn_auto_rollback_check_status,
                QueryToolLocators.btn_auto_rollback)

    def close_data_grid(self):
        self.driver.switch_to_default_content()
        xpath = "//*[@id='dockerContainer']/div/div[3]/div/div[2]/div[1]"
        self.click_element(self.find_by_xpath(xpath))

    def remove_server(self, server_config):
        self.driver.switch_to.default_content()
        server_to_remove = self.find_by_xpath(
            "//*[@id='tree']//*[.='" + server_config['name'] +
            "' and @class='aciTreeItem']")
        self.driver.execute_script(
            self.js_executor_scrollintoview_arg, server_to_remove)
        self.click_element(server_to_remove)
        object_menu_item = self.find_by_partial_link_text("Object")
        self.click_element(object_menu_item)
        delete_menu_item = self.find_by_partial_link_text("Remove Server")
        self.click_element(delete_menu_item)
        self.click_modal('Yes')

    def select_tree_item(self, tree_item_text):
        item = self.find_by_xpath(
            "//*[@id='tree']//*[contains(text(), '" + tree_item_text + "')]"
            "/parent::span[@class='aciTreeItem']")
        self.driver.execute_script(self.js_executor_scrollintoview_arg, item)
        # unexpected exception like element overlapping, click attempts more
        # than one time
        attempts = 3
        while attempts > 0:
            try:
                item.click()
                break
            except Exception as e:
                attempts -= 1
                time.sleep(.4)
                if attempts == 0:
                    raise e

    def click_a_tree_node(self, element_name, list_of_element):
        """It will click a tree node eg. server, schema, table name etc
        will take server name and list of element where this node lies"""
        operation_status = False
        elements = list_of_element = self.find_by_xpath_list(
            list_of_element)
        if len(elements) > 0:
            index_of_element = self.get_index_of_element(
                elements, element_name)
            if index_of_element >= 0:
                self.driver.execute_script(
                    self.js_executor_scrollintoview_arg,
                    list_of_element[index_of_element])
                self.wait_for_elements_to_appear(
                    self.driver, list_of_element[index_of_element])
                time.sleep(1)
                list_of_element[index_of_element].click()
                operation_status = True
            else:
                print("{ERROR} - The required element with name: " + str(
                    element_name) +
                    " is not found in function click_a_tree_node, "
                    "so click operation is not performed")
        else:
            print("{ERROR} - The element list passed to function "
                  "click_a_tree_node seems empty")
        return operation_status

    def toggle_open_servers_group(self):
        """This will open Servers group to display underlying nodes"""
        is_expanded = False
        self.wait_for_spinner_to_disappear()
        if self.check_if_element_exist_by_xpath(
                TreeAreaLocators.server_group_node):
            if self.get_expansion_status_of_node(
                    TreeAreaLocators.server_group_node_exp_status):
                is_expanded = True
            else:
                webdriver.ActionChains(self.driver).double_click(
                    self.find_by_xpath(
                        TreeAreaLocators.
                        server_group_node)).perform()
                if self.check_if_element_exist_by_xpath(
                        TreeAreaLocators.server_group_sub_nodes):
                    is_expanded = True
                else:
                    print(
                        "(toggle_open_servers_group)The Server Group "
                        "node is clicked to expand but it is not expanded",
                        file=sys.stderr)
        else:
            print("The Server Group node is not visible",
                  file=sys.stderr)
        return is_expanded

    def expand_server_node(self, server_name, server_password):
        """will expand a particular server node"""
        server_node_expansion_status = False
        if self.toggle_open_servers_group():
            if self.wait_for_elements_to_appear(
                self.driver,
                    TreeAreaLocators.server_group_sub_nodes):
                subnodes_of_servers = self.find_by_xpath_list(
                    TreeAreaLocators.server_group_sub_nodes)
                subnodes_of_servers_expansion_status = \
                    self.find_by_xpath_list(
                        TreeAreaLocators.
                        server_group_sub_nodes_exp_status)
                index_of_server_node = self.get_index_of_element(
                    subnodes_of_servers, server_name)

                server_node_expansion_status = self.click_expand_server_node(
                    subnodes_of_servers_expansion_status,
                    index_of_server_node,
                    subnodes_of_servers,
                    server_name,
                    server_password)
        else:
            print(
                "(expand_server_node) The Servers node is"
                " not expanded",
                file=sys.stderr)
        return server_node_expansion_status

    def click_expand_server_node(self, subnodes_of_servers_expansion_status,
                                 index_of_server_node, subnodes_of_servers,
                                 server_name, server_password):
        """
        Method actually clicks on server node to expand
        :param subnodes_of_servers_expansion_status:
        :param index_of_server_node:
        :param subnodes_of_servers:
        :param server_name:
        :param server_password:
        :return: True is click action is successful & server node expanded
        """
        server_node_expansion_status = False
        if not self.check_server_is_connected(
                index_of_server_node):
            if self.click_and_connect_server(
                subnodes_of_servers[index_of_server_node],
                    server_password):
                server_node_expansion_status = True
            else:
                print(
                    "(expand_server_node)The server node is "
                    "not expanded",
                    file=sys.stderr)
        else:
            if not self.get_expansion_status_of_node_element(
                subnodes_of_servers_expansion_status[
                    index_of_server_node]):
                webdriver.ActionChains(self.driver).double_click(
                    subnodes_of_servers[
                        index_of_server_node]).perform()
                if self.wait_for_elements_to_appear(
                    self.driver, TreeAreaLocators.
                        sub_nodes_of_a_server_node(server_name),
                        30):
                    server_node_expansion_status = True
            else:
                server_node_expansion_status = True
        return server_node_expansion_status

    def expand_databases_node(self, server_name, server_password):
        """will expand databases node under server node"""
        databases_node_expanded = False
        if self.expand_server_node(server_name, server_password):
            if self.wait_for_elements_to_appear(
                self.driver,
                    TreeAreaLocators.sub_nodes_of_a_server_node(server_name)):
                subnodes_of_server_node = self.find_by_xpath_list(
                    TreeAreaLocators.sub_nodes_of_a_server_node(server_name))
                subnode_of_server_node_exp_status = self.find_by_xpath_list(
                    TreeAreaLocators.sub_nodes_of_a_server_node_exp_status(
                        server_name))
                index_of_databases_node = self.get_index_of_element(
                    subnodes_of_server_node,
                    "Databases")
                time.sleep(2)
                expansion_status = self.get_expansion_status_of_node_element(
                    subnode_of_server_node_exp_status[index_of_databases_node])
                if not expansion_status:
                    databases_node_expanded = \
                        self.click_to_expand_databases_node(
                            subnodes_of_server_node,
                            index_of_databases_node,
                            server_name)
                else:
                    databases_node_expanded = True
        else:
            print("The server/previous nodes not expanded",
                  file=sys.stderr)
        return databases_node_expanded

    def click_to_expand_databases_node(self, subnodes_of_server_node,
                                       index_of_databases_node, server_name):
        """
        Method clicks on databases node of specified server to expand
        :param subnodes_of_server_node:
        :param index_of_databases_node:
        :param server_name:
        :return: True if database node click is successful & expanded
        """
        retry = 5
        databases_node_expanded = False
        while retry > 0:
            webdriver.ActionChains(self.driver).double_click(
                subnodes_of_server_node[
                    index_of_databases_node].find_element_by_xpath(
                    ".//*[@class='aciTreeItem']")
            ).perform()
            if self.wait_for_elements_to_appear(
                self.driver, TreeAreaLocators.
                    sub_nodes_of_databases_node(server_name), 3):
                databases_node_expanded = True
                break
            else:
                retry -= 1
        return databases_node_expanded

    def click_to_expand_database_node(self, sub_nodes_of_databases_node,
                                      index_of_required_db_node,
                                      name_of_database):
        """
        Method clicks on specified database name from expanded databases node
        of server.
        :param sub_nodes_of_databases_node:
        :param index_of_required_db_node:
        :param name_of_database:
        :return: True if particular database click is successful & expanded
        """
        retry = 5
        db_node_expanded_status = False
        self.driver.execute_script(self.js_executor_scrollintoview_arg,
                                   sub_nodes_of_databases_node[
                                       index_of_required_db_node])
        while retry > 0:
            webdriver.ActionChains(self.driver).double_click(
                sub_nodes_of_databases_node[
                    index_of_required_db_node]).perform()
            if self.check_if_element_exist_by_xpath(
                "//div[@class='ajs-header'and text()='INTERNAL SERVER "
                    "ERROR']", 1):
                try:
                    self.click_modal('OK')
                except Exception:
                    pass
                retry -= 1
            else:
                break
        if self.wait_for_elements_to_appear(
            self.driver, TreeAreaLocators.
                sub_nodes_of_database_node(
                name_of_database)):
            db_node_expanded_status = True
        return db_node_expanded_status

    def expand_database_node(self, server_name, server_password,
                             name_of_database):
        """will expand database node under databases node"""
        db_node_expanded_status = False
        if self.expand_databases_node(server_name, server_password):
            sub_nodes_of_databases_node = self.find_by_xpath_list(
                TreeAreaLocators.sub_nodes_of_databases_node(server_name))
            index_of_required_db_node = self.get_index_of_element(
                sub_nodes_of_databases_node,
                name_of_database)
            expansion_status = self.get_expansion_status_of_node_element(
                self.find_by_xpath_list(
                    TreeAreaLocators.
                    sub_nodes_of_databases_node_exp_status(
                        server_name))[
                    index_of_required_db_node])
            if not expansion_status:
                db_node_expanded_status = self.click_to_expand_database_node(
                    sub_nodes_of_databases_node, index_of_required_db_node,
                    name_of_database)
            else:
                db_node_expanded_status = True
        else:
            print("The databases/previous nodes not expanded",
                  file=sys.stderr)
        return db_node_expanded_status

    def toggle_open_schemas_node(self, server_name, server_password,
                                 name_of_database):
        """will expand schemas node under a db node"""
        expansion_status = False
        if self.expand_database_node(server_name, server_password,
                                     name_of_database):
            sub_nodes_db_node = self.find_by_xpath_list(
                TreeAreaLocators.sub_nodes_of_database_node(
                    name_of_database))
            index_of_schemas_node = self.get_index_of_element(
                sub_nodes_db_node, "Schemas")
            expansion_status = self.get_expansion_status_of_node_element(
                self.find_by_xpath_list(
                    TreeAreaLocators.sub_nodes_of_database_node_exp_status(
                        name_of_database))[
                    index_of_schemas_node])
            if not expansion_status:
                self.driver.execute_script(
                    self.js_executor_scrollintoview_arg,
                    sub_nodes_db_node[index_of_schemas_node])
                webdriver.ActionChains(self.driver).double_click(
                    sub_nodes_db_node[index_of_schemas_node]).perform()
                if self.wait_for_elements_to_appear(
                    self.driver, TreeAreaLocators.
                        sub_nodes_of_schemas_node(name_of_database)):
                    expansion_status = True
            else:
                expansion_status = True
        else:
            print(
                "(expand_schemas_node) database/previous nodes "
                "are not expanded",
                file=sys.stderr)
        return expansion_status

    def toggle_open_schema_node(
        self, server_name, server_password,
            name_of_database, name_of_schema_node):
        """will expand schema node under schemas node"""
        expansion_status = False
        if self.toggle_open_schemas_node(
                server_name, server_password, name_of_database):
            sub_nodes_schemas_node = self.find_by_xpath_list(
                TreeAreaLocators.sub_nodes_of_schemas_node(
                    name_of_database))
            index_of_schema_node = self.get_index_of_element(
                sub_nodes_schemas_node,
                name_of_schema_node)
            expansion_status = self.get_expansion_status_of_node_element(
                self.find_by_xpath_list(
                    TreeAreaLocators.sub_nodes_of_schemas_node_exp_status(
                        name_of_database))[
                    index_of_schema_node])
            if not expansion_status:
                self.driver.execute_script(
                    self.js_executor_scrollintoview_arg,
                    sub_nodes_schemas_node[index_of_schema_node])
                webdriver.ActionChains(self.driver).double_click(
                    sub_nodes_schemas_node[index_of_schema_node]).perform()
                if self.wait_for_elements_to_appear(
                    self.driver, TreeAreaLocators.
                        sub_nodes_of_schema_node(name_of_database)):
                    expansion_status = True
            else:
                expansion_status = True
        else:
            print(
                "(expand_schema_node) schema/previous nodes are"
                " not expanded",
                file=sys.stderr)
        return expansion_status

    def toggle_open_tables_node(
        self, server_name, server_password,
            name_of_database, name_of_schema_node):
        """will expand tables node under schema node"""
        node_expanded_successfully = False
        if self.toggle_open_schema_node(
            server_name, server_password, name_of_database,
                name_of_schema_node):
            sub_nodes_of_schema_node = self.find_by_xpath_list(
                TreeAreaLocators.sub_nodes_of_schema_node(
                    name_of_database))
            sub_nodes_of_schema_node_exp_status = self.find_by_xpath_list(
                TreeAreaLocators.sub_nodes_of_schema_node_exp_status(
                    name_of_database))
            index_of_tables_node = self.get_index_of_element(
                sub_nodes_of_schema_node, "Tables")
            expansion_status = self.get_expansion_status_of_node_element(
                sub_nodes_of_schema_node_exp_status[index_of_tables_node])
            if not expansion_status:
                self.driver.execute_script(self.js_executor_scrollintoview_arg,
                                           sub_nodes_of_schema_node[
                                               index_of_tables_node])
                webdriver.ActionChains(self.driver).double_click(
                    sub_nodes_of_schema_node[
                        index_of_tables_node]).perform()
                if self.wait_for_elements_to_appear(
                    self.driver, TreeAreaLocators.
                        sub_nodes_of_tables_node):
                    node_expanded_successfully = True
            else:
                node_expanded_successfully = True
        else:
            print(
                "(expand_tables_node) schema/previous nodes "
                "are not expanded",
                file=sys.stderr)
        return node_expanded_successfully

    def toggle_open_function_node(self):
        """The function will be used for opening Functions node only"""
        node_expanded = False
        attempts = 3

        xpath_for_functions_node = \
            "//span[@class='aciTreeText' and starts-with(text()," \
            "'Functions')]"
        xpath_for_exp = "//div[div[div[div[div[div[div[div[span[span[" \
                        "(@class='aciTreeText') and starts-with(text()," \
                        "'Functions')]]]]]]]]]]"
        xpath_for_button = "//div[span[span[(@class='aciTreeText') " \
                           "and starts-with(text(),'Functions')]]]" \
                           "/span[@class='aciTreeButton']"

        while node_expanded is not True and attempts > 0:
            # get the element which contains 'aria-expanded' info

            xpath_for_refresh_btn = "//li[@class='context-menu-item']" \
                                    "/span[text()='Refresh...']"

            # add code to refresh button, sometime the the collapsing button
            #  is not visible even if there is sub node.
            functions_node_ele = self.find_by_xpath(xpath_for_functions_node)

            webdriver.ActionChains(self.driver).move_to_element(
                functions_node_ele).context_click().perform()
            refresh_btn = self.find_by_xpath(xpath_for_refresh_btn)
            refresh_btn.click()
            time.sleep(.5)

            # get the expansion status
            function_expansion_ele = self.find_by_xpath(xpath_for_exp)

            # look into the attribute and check if it is already expanded or
            #  not
            if function_expansion_ele.get_attribute('aria-expanded') \
                    == 'false':
                # button element of the Function node to open it
                item_button = self.find_by_xpath(xpath_for_button)
                ActionChains(self.driver).click(item_button).perform()
                # Expansion of element on GUI takes sometime, so put small
                # sleep
                time.sleep(.5)
                function_expansion_ele = self.find_by_xpath(
                    xpath_for_exp)
                if function_expansion_ele.get_attribute('aria-expanded') \
                        == 'true':
                    break
                else:
                    attempts -= 1
            else:
                node_expanded = True

    def check_server_is_connected(self, index_of_server):
        """This will check connected status of a server, as connection
        status is contained either in span or div element so checking it"""
        server_connected = False
        try:
            connection_status_elements = self.find_by_xpath_list(
                TreeAreaLocators.server_group_sub_nodes_connected_status)
            span_elements = connection_status_elements[
                index_of_server].find_elements_by_tag_name("span")
            div_elements = connection_status_elements[
                index_of_server].find_elements_by_tag_name("div")

            span_value_of_class_att = ""
            div_value_of_class_att = ""

            if len(span_elements) > 0:
                span_value_of_class_att = \
                    span_elements[0].get_attribute('class')
            if len(div_elements) > 0:
                div_value_of_class_att = \
                    div_elements[0].get_attribute('class')
            if (("aciTreeIcon icon-pg" in span_value_of_class_att or
                 "aciTreeIcon icon-pg" in div_value_of_class_att or
                 "aciTreeIcon icon-ppas" in
                 span_value_of_class_att or
                 "aciTreeIcon icon-ppas" in div_value_of_class_att) and
                    ("aciTreeIcon icon-server-not-connected" not in
                        span_value_of_class_att or
                        "aciTreeIcon icon-server-not-connected" not in
                        div_value_of_class_att)):
                server_connected = True
        except Exception as e:
            print("There is some exception thrown in the function "
                  "check_server_is_connected and is: " + str(e),
                  file=sys.stderr)
        return server_connected

    def click_and_connect_server(self, server_element, password):
        """will connect a server node, will provide the password in the
        respective window"""
        server_connection_status = False
        try:
            webdriver.ActionChains(self.driver).double_click(
                server_element).perform()
            if self.check_if_element_exist_by_xpath(
                    ConnectToServerDiv.ok_button):
                field = self.find_by_xpath(
                    ConnectToServerDiv.password_field)
                self.fill_input(field, password)
                self.find_by_xpath(ConnectToServerDiv.ok_button).click()
                self.wait_for_element_to_disappear(
                    lambda driver: driver.find_element_by_xpath(
                        ConnectToServerDiv.ok_button))
                if self.check_if_element_exist_by_xpath(
                        ConnectToServerDiv.error_message, 2):
                    print(
                        "While entering password in click_and_connect_server "
                        "function, error is occurred : " + str(
                            self.find_by_xpath(
                                ConnectToServerDiv.error_message).text),
                        file=sys.stderr)
                else:
                    server_connection_status = True
        except Exception as e:
            print(
                "There is some exception thrown click_and_connect_server "
                "and is: " + str(
                    e), file=sys.stderr)
        return server_connection_status

    def get_expansion_status_of_node(self, xpath_node):
        """get the expansion status for a node through xpath"""
        node_is_expanded = False
        element = self.find_by_xpath(xpath_node)
        if element.get_attribute("aria-expanded") == 'true':
            node_is_expanded = True
        return node_is_expanded

    def get_expansion_status_of_node_element(self, element):
        """get the expansion status for an element"""
        node_is_expanded = False
        try:
            if element.get_attribute("aria-expanded") == 'true':
                node_is_expanded = True
        except Exception as e:
            print(
                "There is some exception thrown in the function "
                "get_expansion_status_of_node_element and is: " + str(
                    e), file=sys.stderr)
        return node_is_expanded

    def toggle_open_tree_item(self, tree_item_text):
        # 'sleep' here helps in cases where underlying nodes are auto opened.
        # Otherwise, encountered situations where False value is returned
        # even if the underlying node to be clicked was Opened.
        time.sleep(.6)
        item_with_text = self.find_by_xpath(
            TreeAreaLocators.specified_tree_node.format(tree_item_text))

        self.driver.execute_script(self.js_executor_scrollintoview_arg,
                                   item_with_text)

        if item_with_text.find_element_by_xpath(
            ".//ancestor::*[@class='aciTreeLine']").get_attribute(
                "aria-expanded") == 'false':
            item = item_with_text.find_element_by_xpath(
                ".//parent::*[@class='aciTreeItem']")
            ActionChains(self.driver).double_click(item).perform()
        retry = 3
        while retry > 0:
            try:
                WebDriverWait(self.driver, 5).until((lambda item_with_text: (
                    item_with_text.find_element_by_xpath(
                        ".//ancestor::*[@class='aciTreeLine']").
                    get_attribute("aria-expanded") == 'true')))
                break
            except TimeoutException:
                retry -= 1

    def toggle_open_server(self, tree_item_text):
        def check_for_password_dialog_or_tree_open(driver):
            try:
                dialog = driver.find_element_by_id("frmPassword")
            except WebDriverException:
                dialog = None
            try:
                database_node = driver.find_element_by_xpath(
                    "//*[@id='tree']//*[.='Databases']"
                    "/../*[@class='aciTreeButton']")
            except WebDriverException:
                database_node = None

            return dialog is not None or database_node is not None

        self.toggle_open_tree_item(tree_item_text)
        self._wait_for("Waiting for password dialog or tree to open",
                       check_for_password_dialog_or_tree_open)

        try:
            self.driver.find_element_by_id("frmPassword")
            # Enter password here if needed
            self.click_modal('OK')
        except WebDriverException:
            return

    def find_by_xpath(self, xpath):
        return self.wait_for_element(
            lambda driver: driver.find_element_by_xpath(xpath)
        )

    def find_by_id(self, element_id):
        return self.wait_for_element(
            lambda driver: driver.find_element_by_id(element_id)
        )

    def find_by_css_selector(self, css_selector):
        return self.wait_for_element(
            lambda driver: driver.find_element_by_css_selector(css_selector)
        )

    def find_by_partial_link_text(self, link_text):
        return self._wait_for(
            'link with text "{0}"'.format(link_text),
            EC.element_to_be_clickable((By.PARTIAL_LINK_TEXT, link_text))
        )

    def click_element(self, element):
        # driver must be here to adhere to the method contract in
        # selenium.webdriver.support.wait.WebDriverWait.until()
        def click_succeeded(driver):
            try:
                element.click()
                return True
            except WebDriverException:
                return False

        return self._wait_for(
            "clicking the element not to throw an exception", click_succeeded
        )

    def js_send_key(self, field, sele_key):
        keycode = None

        if sele_key in (Keys.RETURN, Keys.ENTER):
            keycode = 13
        elif sele_key == Keys.ARROW_DOWN:
            keycode = 40

        self.driver.execute_script(
            "arguments[0].dispatchEvent(new KeyboardEvent('keydown', "
            "{'keyCode':arguments[1], 'which':arguments[1]}));"
            "arguments[0].dispatchEvent(new KeyboardEvent('keypress', "
            "{'keyCode':arguments[1], 'which':arguments[1]}));"
            "arguments[0].dispatchEvent(new KeyboardEvent('keyup', "
            "{'keyCode':arguments[1], 'which':arguments[1]}));"
            "arguments[0].dispatchEvent(new Event('input'));"
            "arguments[0].dispatchEvent(new Event('change'));",
            field, keycode)

    def js_loose_focus(self, field):
        self.driver.execute_script(
            "arguments[0].dispatchEvent(new Event('blur'));", field)

    def fill_input(self, field, field_content, input_keys=False,
                   key_after_input=Keys.ARROW_DOWN):
        try:
            attempt = 0
            for attempt in range(0, 3):
                field.click()
                break
        except Exception as e:
            time.sleep(.2)
            if attempt == 2:
                raise e
        # Use send keys if input_keys true, else use javascript to set content
        if input_keys:
            backspaces = [Keys.BACKSPACE] * len(field.get_attribute('value'))
            field.send_keys(backspaces)
            field.send_keys(str(field_content))
            self.wait_for_input_by_element(field, field_content)
        else:
            self.driver.execute_script("arguments[0].value = arguments[1]",
                                       field, field_content)
            # keycode 40 is for arrow down
            self.js_send_key(field, Keys.ARROW_DOWN)

            if key_after_input:
                self.js_send_key(field, key_after_input)

    def fill_input_by_field_name(self, field_name, field_content,
                                 input_keys=False,
                                 key_after_input=Keys.ARROW_DOWN,
                                 loose_focus=False):
        field = self.find_by_css_selector(
            "input[name='" + field_name + "']:not(:disabled)")
        self.fill_input(field, field_content, input_keys=input_keys,
                        key_after_input=key_after_input)

        if loose_focus:
            self.js_loose_focus(field)

    def fill_input_by_css_selector(self, css_selector, field_content,
                                   input_keys=False,
                                   key_after_input=Keys.ARROW_DOWN,
                                   loose_focus=False):
        field = self.find_by_css_selector(css_selector)
        self.fill_input(field, field_content, input_keys=input_keys,
                        key_after_input=key_after_input)

        if loose_focus:
            self.js_loose_focus(field)

    def fill_codemirror_area_with(self, field_content, input_keys=False):
        def find_codemirror(driver):
            try:
                driver.switch_to.default_content()
                driver.switch_to_frame(
                    driver.find_element_by_tag_name("iframe"))
                element = driver.find_element_by_css_selector(
                    "#output-panel .CodeMirror")
                if element.is_displayed() and element.is_enabled():
                    return element
            except (NoSuchElementException, WebDriverException):
                return False
        time.sleep(1)
        self.wait_for_query_tool_loading_indicator_to_disappear(12)

        retry = 2
        while retry > 0:
            try:
                self.driver.switch_to.default_content()
                WebDriverWait(self.driver, 10).until(
                    EC.frame_to_be_available_and_switch_to_it(
                        (By.TAG_NAME, "iframe")))
                self.find_by_xpath("//a[text()='Query Editor']").click()

                codemirror_ele = WebDriverWait(
                    self.driver, timeout=self.timeout, poll_frequency=0.01) \
                    .until(find_codemirror,
                           "Timed out waiting for codemirror to appear")
                codemirror_ele.click()
                retry = 0
            except WebDriverException as e:
                print("Exception in filling code mirror {0} ".format(retry),
                      file=sys.stderr)
                print(str(e))
                if retry == 1:
                    raise e
                retry -= 1

        # Use send keys if input_keys true, else use javascript to set content
        if input_keys:
            action = ActionChains(self.driver)
            action.send_keys(field_content)
            action.perform()
        else:
            self.driver.execute_script(
                "arguments[0].CodeMirror.setValue(arguments[1]);"
                "arguments[0].CodeMirror.setCursor("
                "arguments[0].CodeMirror.lineCount(),0);",
                codemirror_ele, field_content)

    def click_tab(self, tab_name):
        WebDriverWait(self.driver, 10).until(EC.element_to_be_clickable(
            (By.XPATH, NavMenuLocators.select_tab_xpath.format(tab_name))))
        click_tab = True
        while click_tab:
            tab = self.find_by_xpath(
                NavMenuLocators.select_tab_xpath.format(tab_name))
            self.click_element(tab)
            if 'wcPanelTabActive' in tab.get_attribute('class'):
                break

    def wait_for_input_by_element(self, element, content):
        def input_field_has_content(driver):
            return str(content) == element.get_attribute('value')

        return self._wait_for(
            "field to contain '" + str(content) + "'", input_field_has_content
        )

    def wait_for_input_field_content(self, field_name, content, wait=1):
        def input_field_has_content(driver):
            element = driver.find_element_by_xpath(
                "//input[@name='" + field_name + "']")

            return str(content) == element.get_attribute('value')

        return self._wait_for(
            "field to contain '" + str(content) + "'", input_field_has_content,
            wait)

    def check_if_element_exist_by_xpath(self, xpath, timeout=5):
        """This function will verify if an element exist and on that basis
        will return True or False. Will handle exception internally"""
        element_found = False
        try:
            WebDriverWait(self.driver, timeout, .01).until(
                EC.visibility_of_element_located((By.XPATH, xpath)))
            element_found = True
        except Exception:
            pass
        return element_found

    def wait_for_element(self, find_method_with_args):
        def element_if_it_exists(driver):
            try:
                element = find_method_with_args(driver)
                if element.is_displayed():
                    return True
            except (NoSuchElementException, StaleElementReferenceException):
                return False

        self._wait_for("element to exist", element_if_it_exists)
        return find_method_with_args(self.driver)

    def wait_for_element_to_disappear(self, find_method_with_args, timeout=5):
        def element_if_it_disappears(driver):
            try:
                element = find_method_with_args(driver)
                if element.is_displayed() and element.is_enabled():
                    return False

                return True
            except (NoSuchElementException, StaleElementReferenceException):
                return True

        return self._wait_for("element to disappear",
                              element_if_it_disappears, timeout)

    def wait_for_reloading_indicator_to_disappear(self):
        def reloading_indicator_has_disappeared(driver):
            try:
                driver.find_element_by_id("reloading-indicator")
                return False
            except NoSuchElementException:
                return True

        self._wait_for("reloading indicator to disappear",
                       reloading_indicator_has_disappeared)

    def wait_for_spinner_to_disappear(self):
        def spinner_has_disappeared(driver):
            try:
                driver.find_element_by_id("pg-spinner")
                return False
            except NoSuchElementException:
                return True

        self._wait_for("spinner to disappear", spinner_has_disappeared, 20)

    def wait_for_query_tool_loading_indicator_to_disappear(self, timeout=20):
        def spinner_has_disappeared(driver):
            try:
                spinner = driver.find_element_by_css_selector(
                    "#editor-panel .pg-sp-container"
                )
                return "d-none" in spinner.get_attribute("class")
            except NoSuchElementException:
                # wait for loading indicator disappear animation to complete.
                time.sleep(0.5)
                return True

        self._wait_for(
            "spinner to disappear", spinner_has_disappeared, timeout)

    def wait_for_query_tool_loading_indicator_to_appear(self):
        status = self.check_if_element_exist_by_xpath(
            "//div[@id='editor-panel']//"
            "div[@class='pg-sp-container sql-editor-busy-fetching']", 1)
        return status

    def wait_for_app(self):
        def page_shows_app(driver):
            if driver.title == self.app_config.APP_NAME:
                return True
            else:
                driver.refresh()
                return False

        self._wait_for("app to start", page_shows_app, self.app_start_timeout)

    def wait_for_element_to_reload(self, element_selector):
        WebDriverWait(self.driver, 20) \
            .until(EC.staleness_of(element_selector(self.driver)))
        WebDriverWait(self.driver, 20) \
            .until_not(EC.staleness_of(element_selector(self.driver)))

        return element_selector(self.driver)

    def _wait_for(self, waiting_for_message, condition_met_function,
                  timeout=5):
        if timeout is None:
            timeout = self.timeout
        return WebDriverWait(self.driver, timeout, 0.01).until(
            condition_met_function,
            "Timed out waiting for " + waiting_for_message
        )

    def wait_for_elements(self, find_method_with_args):
        """Using xpath, it will wait for elements"""
        def element_if_it_exists(driver):
            try:
                element = find_method_with_args(driver)
                if len(element) > 0 and element[0].is_displayed() and element[
                        0].is_enabled():
                    return element
            except NoSuchElementException:
                return False

        return self._wait_for("element to exist", element_if_it_exists)

    def wait_for_elements_to_appear(self, driver, locator, time_value=20):
        """This will wait until list of elements or an element is visible,
        The time out value is userdefined"""
        elements_located_status = False
        try:
            if WebDriverWait(driver, time_value).until(
                    EC.visibility_of_any_elements_located((
                        By.XPATH, locator))):
                elements_located_status = True
        except Exception:
            pass
        return elements_located_status

    def find_by_xpath_list(self, xpath):
        """This will find out list of elements through a single xpath"""
        return self.wait_for_elements(
            lambda driver: driver.find_elements_by_xpath(xpath))

    def get_index_of_element(self, element_list, target_string):
        """it will return index of an element from provided element list"""
        index_of_required_server = -1
        if len(element_list) > 0:
            for index, element in enumerate(element_list):
                if element.text.startswith(target_string) and (
                        target_string in element.text):
                    index_of_required_server = index
                    break
        else:
            print("There seems no record in the provided element list")
        return index_of_required_server

    def set_switch_box_status(self, switch_box, required_status):
        """it will change switch box status to required one. Two elements
        of the switch boxes are to be provided i) button which is needed to
         toggle ii) Yes for True or No for False"""
        status_changed_successfully = False
        switch_box_element = self.find_by_xpath(switch_box)

        if required_status == 'Yes':
            status_changed_successfully = \
                self.toggle_switch_box(switch_box_element,
                                       expected_attr_in_class_tag='success',
                                       unexpected_attr_in_class_tag='off')
        else:
            status_changed_successfully = \
                self.toggle_switch_box(switch_box_element,
                                       expected_attr_in_class_tag='off',
                                       unexpected_attr_in_class_tag='success')
        return status_changed_successfully

    def toggle_switch_box(self, switch_box_ele, expected_attr_in_class_tag,
                          unexpected_attr_in_class_tag):
        """
        Method toggles switch box status using attributes from class tag
        :param switch_box_ele:
        :param expected_attr_in_class_tag: e.g. 'off', success
        :param unexpected_attr_in_class_tag: e.g. 'off', success
        :return: True if class tag attribute is to expected attribute value
        in class tag.
        """
        status_changed = False
        if unexpected_attr_in_class_tag in switch_box_ele.get_attribute(
                "class"):
            switch_box_ele.click()
            time.sleep(1)
            if expected_attr_in_class_tag in switch_box_ele.get_attribute(
                    "class"):
                status_changed = True
            else:
                print(
                    "(set_switch_box_status)Clicked the "
                    "element to change its status but "
                    "it did not changed",
                    file=sys.stderr)
        elif expected_attr_in_class_tag in switch_box_ele.get_attribute(
                "class"):
            status_changed = True
        return status_changed

    def retry_click(self, click_locator, verify_locator):
        click_status = False
        attempt = 0

        while click_status is not True and attempt < 10:
            try:
                element = self.driver.find_element(*click_locator)
                element.click()
                WebDriverWait(self.driver, 10).until(
                    EC.visibility_of_element_located(verify_locator))
                click_status = True
            except Exception:
                attempt += 1
        return click_status

    def paste_values(self, el=None):
        """
        Function paste values in scratch pad
        :param el:
        """
        actions = ActionChains(self.driver)
        if "platform" in self.driver.capabilities:
            platform = (self.driver.capabilities["platform"]).lower()
        elif "platformName" in self.driver.capabilities:
            platform = (self.driver.capabilities["platformName"]).lower()
        if el:
            # Must step
            el.click()
            if 'mac' in platform:
                # FF step
                el.send_keys(Keys.COMMAND + "v")
                # Chrome Step
                actions.key_down(Keys.SHIFT)
                actions.send_keys(Keys.INSERT)
                actions.key_up(Keys.SHIFT)
                actions.perform()
            else:
                el.send_keys(Keys.CONTROL + "v")

    def wait_for_element_to_be_visible(self, driver, xpath, time_value=20):
        """This will wait until an element is visible on page"""
        element_located_status = False
        try:
            if WebDriverWait(driver, time_value).until(
                    EC.visibility_of_element_located((By.XPATH, xpath))):
                element_located_status = True
        except TimeoutException:
            element_located_status = False
        return element_located_status


import pdfplumber

class pdf_scraper():

    def __init__(self, pdf_file, print=False):
        self.pdf_path = pdf_file
        self.print = print

    def extract_all_text(self):
        # Returns all text from pdf as a string
        data = None
        with pdfplumber.open(self.pdf_path) as pdf:
            for page_num, page in enumerate(pdf.pages):
                page_data = page.extract_text()
                data += page_data
                if print:
                    print("------- Page {}-----------".format((page_num + 1)))
        pdf.close()
        return data

    def extract_all_tables(self):
        # Returns all data from pdf as a dictionary
        data = {}
        with pdfplumber.open(self.pdf_path) as pdf:
            # Loop through all the pages in the pdf
            for page_num, page in enumerate(pdf.pages):
                print("Scraping tables from page {} of {}".format(page_num + 1, self.pdf_path))
                # Extract all tables from the page
                page_data = page.extract_tables()

                # Data is in format of list, in a list
                # so we must iterate through
                for item in page_data:
                    for subitem in item:
                        # Raw data - formatting to give proper
                        # key value pairs in dictionary
                        formatted = subitem[0].strip()
                        while "\n" in formatted:
                            formatted = formatted.replace("\n", "")
                        split = formatted.split(":", 1)
                        # Ignoring cases we don't need
                        if len(split) == 2:
                            data[split[0]] = split[1]
        print("Finished scraping tables from all pages in {}\n".format(self.pdf_path))
        pdf.close()

        # Output the data if selected
        if self.print:
            for k,v in data.items():
                print(k, v)

        return data


"""Serverless datalake monitoring constructs"""
import json
# pylint: disable=unused-variable, too-many-locals
import os
import subprocess as sp

from aws_cdk import (
    aws_lambda as lambda_,
    aws_lambda_event_sources as lambda_event_source,
    aws_events as events,
    aws_events_targets as targets,
    aws_iam as iam,
    aws_glue as glue,
    aws_sns as sns,
    aws_secretsmanager as secrets, Stack, Duration
)
from constructs import Construct

import config as cf
from common.utils import select_artifacts

MONITOR_LAMBDA_ASSETS = {"zips": "*.zip", "lib_dir": "libs", "monitor_lambda": "handler.py"}

class DataLakeMonitoringStack(Stack):
    """Construct containing resources for Data Lake monitoring"""

    def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
        """Create construct"""
        super().__init__(scope, construct_id, **kwargs)

        lambda_timeout_seconds = 180
        path_monitoring_lambda = os.path.join(cf.PATH_SRC, "datalake_monitoring")
        monitor_db = cf.MONITOR_DB
        monitor_table = cf.MONITOR_TABLE

        # Secret for Monitoring
        secrets_kv = {
            f"{cf.SLACK_WEBHOOK_SECRET_NAME}": "to-be-updated"
        }

        monitoring_secret = secrets.Secret(
            self,
            id="monitoring-secret",
            secret_name=cf.MONITOR_SECRET_MANAGER,
            secret_string_beta1=secrets.SecretStringValueBeta1.from_unsafe_plaintext(f"{json.dumps(secrets_kv)}")
        )

        # Create SNS Topic to fork out subscriptions
        self.dl_monitor_sns_topic = sns.Topic(
            self,
            id="dl-monitor-sns",
            display_name="datalake-monitor-sns",
            topic_name=cf.MONITOR_SNS_TOPIC,
        )

        # Lambda layer - aws-wrangler
        sp.call(["make", "bundle"], cwd=path_monitoring_lambda)

        wrangler_layer = lambda_.LayerVersion(
            self,
            id="aws-wrangler",
            code=lambda_.Code.from_asset(
                os.path.join(path_monitoring_lambda, f"libs/{cf.WRANGLER_ASSET}")
            ),
        )

        sp.call(["make", "clean"], cwd=path_monitoring_lambda)

        monitoring_lambda = lambda_.Function(
            self,
            id="datalake-monitoring-lambda",
            handler="handler.handler",
            runtime=lambda_.Runtime.PYTHON_3_10,
            code=lambda_.Code.from_asset(
                path=path_monitoring_lambda,
                exclude=select_artifacts(
                    artifacts=MONITOR_LAMBDA_ASSETS, keep_artifact="monitor_lambda"
                ),
            ),
            function_name="datalake-monitoring-lambda",
            environment={
                "REGION": cf.REGION,
                "SECRET_MGR": cf.MONITOR_SECRET_MANAGER,
                "MONITOR_S3": cf.S3_MONITOR_BUCKET,
                "MONITOR_DATABASE": cf.MONITOR_DB,
                "MONITOR_TABLE": cf.MONITOR_TABLE,
            },
            layers=[wrangler_layer],
            memory_size=128,
            timeout=Duration.seconds(lambda_timeout_seconds),
        )

        monitoring_secret.grant_read(monitoring_lambda)

        # Create an SNS event source for Lambda
        sns_event_source = lambda_event_source.SnsEventSource(self.dl_monitor_sns_topic)

        # Add SNS event source to the Lambda function
        monitoring_lambda.add_event_source(sns_event_source)

        # Glue Events to SNS

        # Create Event Rule to capture Glue Job
        glue_job_rule = events.Rule(
            self,
            id="serverless-event-rule-capture-glue-state",
            description="For any Glue Job or Crawler, capture if it was a failure or success",
            rule_name="glue-monitor-rule",
            enabled=True,
            event_pattern=events.EventPattern(
                detail_type=["Glue Job State Change", "Glue Crawler State Change"],
                detail={
                    "state": ["TIMEOUT", "FAILED", "SUCCEEDED", "STOPPED", "Failed", "Succeeded"]
                },
            ),
            # targets=[targets.SqsQueue(self.central_state_queue)],
            targets=[targets.SnsTopic(self.dl_monitor_sns_topic)],
        )

        # SLACK NOTIFICATION

        # Secret for Monitoring
        # monitor_secret = secrets.Secret.from_secret_name(
        monitor_secret = secrets.Secret.from_secret_name_v2(
            self, id="monitor-stack-secret", secret_name=cf.SLACK_WEBHOOK_SECRET_NAME
        )

        '''
        Monitoring : Database and Table creation
        Create DB, Table
        '''

        # Monitoring Database
        monitor_db = glue.CfnDatabase(
            self,
            id="monitor-db",
            catalog_id=cf.ACCOUNT,
            database_input=glue.CfnDatabase.DatabaseInputProperty(
                name=cf.MONITOR_DB,
                description="Monitoring DB",
                location_uri=f"s3://{cf.S3_MONITOR_BUCKET}/{cf.MONITOR_DB}/",
            ),
        )

        # Monitoring Table
        service_metrics_columns = [
            {"name": column_name, "type": "string", "comment": ""}
            for column_name in [
                "error_message",
                "event_type",
                "exception_details",
                "retry_attempts",
                "service_name",
                "service_request_id",
                "service_type",
                "timestamp",
            ]
        ]

        service_metrics_table = glue.CfnTable.TableInputProperty(
            description="Monitor Table Attributes",
            name=cf.MONITOR_TABLE,
            parameters={
                "classification": "json",
                "has_encrypted_data": "false",
                "projection.enabled": "true",
                "projection.exported_on.type": "date",
                "projection.exported_on.range": "2021-07-01,2021-12-31",
                "projection.exported_on.format": "yyyy-MM-dd",
                "projection.exported_on.interval": "1",
                "projection.exported_on.interval.unit": "DAYS",
            },
            partition_keys=[{"name": "exported_on", "type": "date", "comment": "Day of event"}],
            storage_descriptor=glue.CfnTable.StorageDescriptorProperty(
                columns=service_metrics_columns,
                input_format="org.apache.hadoop.mapred.TextInputFormat",
                output_format="org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
                compressed=True,
                location=f"s3://{cf.S3_MONITOR_BUCKET}/{cf.MONITOR_DB}/{cf.MONITOR_TABLE}/",
                serde_info=glue.CfnTable.SerdeInfoProperty(
                    serialization_library="org.openx.data.jsonserde.JsonSerDe"
                ),
            ),
            table_type="EXTERNAL_TABLE",
        )

        # Create Glue Table
        monitor_table = glue.CfnTable(
            self,
            id="monitor-table",
            catalog_id=cf.ACCOUNT,
            database_name=cf.MONITOR_DB,
            table_input=service_metrics_table,
        )

        monitor_table.add_depends_on(monitor_db)

        # Policy for Lambda to create or replace view. Also update Glue and Athena artifacts
        monitoring_lambda.add_to_role_policy(
            iam.PolicyStatement(
                effect=iam.Effect.ALLOW,
                actions=[
                    "athena:UpdateDataCatalog",
                    "athena:GetDataCatalog",
                    "glue:DeleteTable",
                    "glue:CreateTable",
                    "glue:GetDatabases",
                    "glue:GetSchema",
                    "glue:GetTable",
                ],
                resources=[
                    f"arn:aws:glue:{cf.REGION}:{cf.ACCOUNT}:database/{cf.MONITOR_DB}",
                    f"arn:aws:glue:{cf.REGION}:{cf.ACCOUNT}:schema/{cf.MONITOR_DB}",
                    f"arn:aws:glue:{cf.REGION}:{cf.ACCOUNT}:table/{cf.MONITOR_DB}/*",
                    f"arn:aws:glue:{cf.REGION}:{cf.ACCOUNT}:catalog",
                    f"arn:aws:athena:{cf.REGION}:{cf.ACCOUNT}:datacatalog/AwsDataCatalog",
                    f"arn:aws:athena:{cf.REGION}:{cf.ACCOUNT}:workgroup/primary",
                ],
            )
        )

        # S3 Access
        monitoring_lambda.role.add_to_policy(
            iam.PolicyStatement(
                effect=iam.Effect.ALLOW,
                actions=[
                    "s3:PutObject*",
                    "s3:Abort*",
                    "s3:DeleteObject*",
                    "s3:GetObject*",
                    "s3:GetBucket*",
                    "s3:List*",
                    "s3:Head*"
                ],
                resources=[
                    f"arn:aws:s3:::{cf.S3_MONITOR_BUCKET}",
                    f"arn:aws:s3:::{cf.S3_MONITOR_BUCKET}/{cf.MONITOR_DB}",
                    f"arn:aws:s3:::{cf.S3_MONITOR_BUCKET}/{cf.MONITOR_DB}/*"
                ],
            )
        )

        # Database and Tables Access
        monitoring_lambda.role.add_to_policy(
            iam.PolicyStatement(
                effect=iam.Effect.ALLOW,
                actions=[
                    "glue:CreateTable",
                    "glue:CreatePartition",
                    "glue:UpdatePartition",
                    "glue:UpdateTable",
                    "glue:DeleteTable",
                    "glue:DeletePartition",
                    "glue:BatchCreatePartition",
                    "glue:Get*",
                    "glue:BatchGet*"
                ],
                resources=[
                    f"arn:aws:glue:{cf.REGION}:{cf.ACCOUNT}:catalog",
                    f"arn:aws:glue:{cf.REGION}:{cf.ACCOUNT}:database/{cf.MONITOR_DB}",
                    f"arn:aws:glue:{cf.REGION}:{cf.ACCOUNT}:table/{cf.MONITOR_DB}/{cf.MONITOR_TABLE}",
                    f"arn:aws:glue:{cf.REGION}:{cf.ACCOUNT}:schema/{cf.MONITOR_DB}",
                    f"arn:aws:glue:{cf.REGION}:{cf.ACCOUNT}:catalog",
                    f"arn:aws:athena:{cf.REGION}:{cf.ACCOUNT}:datacatalog/AwsDataCatalog",
                    f"arn:aws:athena:{cf.REGION}:{cf.ACCOUNT}:workgroup/primary",
                ],
            )
        )


import os

from dotenv import load_dotenv

from lof.env_vars import path_for_env_file, set_env_variables

current_dir = os.path.dirname(os.path.abspath(__file__))
test_json_vars = os.path.join(current_dir, "test_data/vars.json")
test_env_vars = os.path.join(current_dir, "test_data/.env")
lof_dir = ".lof"

def test_path_for_env_file():
    expected = ".lof/.env"
    os.makedirs(lof_dir, exist_ok=True)
    assert path_for_env_file(test_json_vars, lof_dir) == expected
    # check no issues
    assert load_dotenv(expected) is True

def test_set_env_variables_json():
    # check no issues
    assert set_env_variables(test_json_vars) is None
    assert os.environ["DB_USER"] == "test"

def test_set_env_variables_env():
    # check no issues
    assert set_env_variables(test_env_vars) is None
    assert os.environ["TEST_ENV"] == "DB_NAME_TEST"


import json
from basics import Creature, KEYWORDS

# Load json (http://mtgjson.com/)
json_file = open("SOI.json")
set_json = json.load(json_file)["cards"]
for i in range(len(set_json)):
    for unwanted in ["artist", "flavor", "id", "imageName", "layout",
                     "multiverseid", "number", "type"]:
        try:
            set_json[i].pop(unwanted)
        except KeyError:
            continue
creature_jsons = [card for card in set_json if "Creature" in card["types"]]
for creature_json in creature_jsons:
    try:
        creature_json["keywords"] = [kw for kw in KEYWORDS if \
                                     kw in creature_json["text"]]
    except KeyError:
        continue
creature_list = [Creature(creature_json) for creature_json in creature_jsons]


import psycopg2
from config import config

def create_tables():
    """ create tables in the PostgreSQL database"""
    commands = """CREATE TABLE currency (
                base_currency TEXT NOT NULL,
                target_currency TEXT NOT NULL,
                rate NUMERIC,
                rate_date DATE,
                etl_modified_date TIMESTAMP,
                PRIMARY KEY (rate_date, etl_modified_date))"""
    conn = None
    try:
        # read the connection parameters
        params = config()
        # connect to the PostgreSQL server
        conn = psycopg2.connect(**params)
        cur = conn.cursor()
        # create table one by one
        cur.execute(commands)
        # close communication with the PostgreSQL database server
        cur.close()
        # commit the changes
        conn.commit()
    except (Exception, psycopg2.DatabaseError) as error:
        print(error)
    finally:
        if conn is not None:
            conn.close()


from django import forms
from .models import Recipe

class RecipeForm(forms.ModelForm):
    class Meta:
        model = Recipe
        fields = [
            'title',
            'summary',
            'preparation_time',
            'cooking_time',
            'serves',
            'freezable',
            'skill_level',
            'star_rating',
            'main_image',
            'ingredients',
            'instructions',
            'image_a',
            'image_b',
            'image_c',
        ]


from amplpy import AMPL
import numpy as np

ampl = AMPL()
ampl.set_option("solver", "cplex")
ampl.read("ext/1D-regression.increasing.smooth.model")

def nonampl(x, y, lam):
    n = len(x)
    ampl.param["datax"] = x
    ampl.param["datay"] = y
    ampl.param["oneperlambda"] = 1.0 / lam
    ampl.param["cvset"] = np.zeros(n)
    ampl.solve()
    ampl.get_variable("p").get_values().to_pandas()