import os
import subprocess
import pycurl
from bs4 import BeautifulSoup
from googlesearch import search
import xml.etree.cElementTree as ET
import ipaddress
from pathlib import Path
import re
import argparse

#todo
#1: make it so SAMBA scan will run based on port numbers open, 139, 445
#2: Fix nikto scan output

#notes:
#1: user input
#2: create md file for report
#3: scan subnet / ip
#4: add active ones to section of report
#5: nmap scan all active IPs
#6: add nmap scans in md format to md report, create a section for each IP
#7: do exploit db search for all services and version numbers running on machines
#8: add exploit-db results to report for each device
#9: attempt inital access with stuff like anonymous login to ftp or samba

def main():
    #get user input
    #get ip or subnet to scan
    print('''                    (                   (                  (
                 )  )\ )             )  )\ )  *   )     )  )\ )
 (    (   (   ( /( (()/(     (    ( /( (()/(` )  /(  ( /( (()/(
 )\   )\  )\  )\()) /(_))    )\   )\()) /(_))( )(_)) )\()) /(_))
((_) ((_)((_)((_)\ (_))     ((_) ((_)\ (_)) (_(_()) ((_)\ (_))
| __|\ \ / /  / (_)| |     _ | ||__ (_)/ __||_   _||__ (_)| _ \
| _|  \ V /   | |  | |__  | || | |_ \  \__ \  | |   |_ \  |   /
|___|  \_/    |_|  |____|  \__/ |___/  |___/  |_|  |___/  |_|_\

''')
    print("Created by: S1n1st3r")

    activeIPs = []
    parser = argparse.ArgumentParser()
    parser.add_argument("-t", "--type",
                    dest="type",
                    help="The type of scan to run, 1 for subnet, 2 for single IP",
                    action='store',
                    required=True)
    parser.add_argument("-i", "--ip",
                    dest="ip",
                    help="ip address/subnet (without the /24) to scan",
                    action='store',
                    required=True)
    parser.add_argument("-o", "--output",
                    dest="output",
                    help="output file name",
                    action='store')
    parser.add_argument("-s", "--stealth",
                    dest="stealth",
                    help="enable stealth mode, requires root",
                    action='store_true')
    parser.add_argument("-a", "--arp",
                    dest="arp",
                    help="enable arp scan instead of ping sweep",
                    action='store_true')

    args = parser.parse_args()
    stealth = args.stealth

    if args.type == "1":
        userInput = args.ip
        if ip_is_valid(userInput):
            print("IP is valid")
            if args.arp:
                print("ARP scan enabled")
                activeIPs = scanARP()
            else:
                activeIPs = scanSubnet(userInput)
        else:
            print("Invalid IP address!")
            quit()
    elif args.type == "2":
        userInput = args.ip
        if ip_is_valid(userInput):
            if ping_ip(userInput):
                print("IP is active")
                activeIPs = [userInput]
            else:
                print("IP is not active")
                quit()
        else:
            print("Invalid IP address!")
            quit()
    else:
        print("Invalid input")
        return

    #create md file for report
    #if output is not specified, use default name
    if args.output is None:
        outputFile = "notes.md"
    else:
        outputFile = args.output
    notesFile = open(outputFile, "w")
    notesFile.write("
Notes for " + userInput + " scan
\n
")
    notesFile.close()
    #add active ones to section of report
    notesFile = open(outputFile, "a")
    notesFile.write("\n
Active IPs
\n
")
    for ip in activeIPs:
        notesFile.write(ip + "\n
")
    notesFile.close()
    #nmap scan all active IPs
    #add nmap scans in md format to md report, create a section for each IP
    listOfServices = nmapScan(activeIPs, outputFile, stealth)
    listOfPorts = getPorts()
    os.remove("temp.xml")
    #attempt inital access with stuff like anonymous login to ftp or samba
    print("Running service enumeration on active IPs")
    for service in listOfServices:
        #if listOfServices contains smb, try smb login
        if "smb" in service or "SMB" in service or '139' in listOfPorts or '445' in listOfPorts:
            for ip in activeIPs:
                SAMBAcheck(ip, outputFile)
        #if listOfServices contains ftp, try ftp login
        if "ftp" in service or "FTP" in service:
            for ip in activeIPs:
                ftpCheck(ip, outputFile)
        #if listOfServices contains http, try http check
        if "http" in service or "HTTP" in service or '80' in listOfPorts or '443' in listOfPorts or '8080' in listOfPorts:
            for ip in activeIPs:
                httpCheck(ip, listOfPorts, outputFile)
        #if listOfServices contains ssh, try ssh check
        if "ssh" in service or "SSH" in service:
            for ip in activeIPs:
                sshCheck(ip, outputFile)
        #if listOfServices contains telnet, try telnet check
        if "telnet" in service or "TELNET" in service:
            for ip in activeIPs:
                telnetCheck(ip, outputFile)
        #if listOfServices contains snmp, try snmp check
        if "snmp" in service or "SNMP" in service:
            for ip in activeIPs:
                snmpCheck(ip, outputFile)
        #if mysql is in listOfServices, try mysql check
        if "mysql" in service or "MYSQL" in service:
            for ip in activeIPs:
                mysqlCheck(ip, outputFile)
        #if listOfServices contains icmp, try icmp check
        if "icmp" in service or "ICMP" in service:
            for ip in activeIPs:
                icmpCheck(ip, outputFile)
        #if listOfServices contains dns, try dns check
        if "dns" in service or "DNS" in service:
            for ip in activeIPs:
                dnsCheck(ip, outputFile)
        #if listOfServices contains smtp, try smtp check
        if "smtp" in service or "SMTP" in service:
            for ip in activeIPs:
                smtpCheck(ip, outputFile)
        #if listOfServices contains pop3, try pop3 check
        if "pop3" in service or "POP3" in service:
            for ip in activeIPs:
                pop3Check(ip, outputFile)
    #do exploit db search for all services and version numbers running on machines
    #add exploit-db results to report for each device
    print("Running exploit-db search on active IPs")
    searchExploitDB(listOfServices, outputFile)
    print("Completed successfully")

# this function will validate a given IP address
# and return True or False based on the validation result
def ip_is_valid(ip):
    try:
        ipaddress.IPv4Network(ip)
        return True
    except Exception as e:
        return False

def ping_ip(ip):
    #ping IP
    DEVNULL = open(os.devnull, 'w')
    try:
        response = os.popen(f"ping -c 2 {ip}").read()
        if "Received = 4" in response:
            return False
        else:
            return True
    except Exception:
            return False

def scanSubnet(subnet):
    DEVNULL = open(os.devnull,'w')
    list = []
    for ping in range(1,255):
        address = subnet[:-1] + str(ping)
        res = subprocess.call(['ping', '-c', '1', address], stdout=DEVNULL)
        if res == 0:
            print("ping to", address, "OK")
            list.append(address)
        elif res == 2:
            print("no response from", address)
        else:
            print("ping to", address, "failed!")
    return list

def scanARP():
    results = []
    os.system('arp -a > temp')
    #get just the ip address from the arp scan output
    pattern = re.compile("(?:[0-9]{1,3}\.){3}[0-9]{1,3}")

    for i, line in enumerate(open('temp')):
        for match in re.finditer(pattern, line):
            results.append(match.group())

    print(results) #remove this
    return results

def getServiceListOutput():
    servicesList = []
    tree = ET.parse('temp.xml')
    root = tree.getroot()
    for item in list(root):
        if item.tag == 'host':
            for child in list(item):
                if child.tag == 'ports':
                    for port in list(child):
                        if port.tag == 'port':
                            for service in list(port):
                                if service.tag == 'service':
                                    try:
                                        servicesList.append(service.attrib['product'] + ' ' + service.attrib['version'])
                                    except KeyError:
                                        pass
    return servicesList

def getPorts():
    portsList = []
    tree = ET.parse('temp.xml')
    root = tree.getroot()
    for item in list(root):
        if item.tag == 'host':
            for child in list(item):
                if child.tag == 'ports':
                    for port in list(child):
                        if port.tag == 'port':
                                try:
                                    portsList.append(port.attrib['portid'])
                                except KeyError:
                                    pass
    return portsList

def cleanMDfile():
    datafile = []
    # if the temp.md file doesn't exist, create an empty file to avoid errors
    tmp_file = Path('temp.md')
    tmp_file.touch(exist_ok=True)
    file = open('temp.md', 'r')
    # delete matching content and 326 lines after
    currentLine = 0
    content = ""
    datafile = file.readlines()
    for line in datafile:
        if content in line:
            lines = currentLine
            while currentLine < (lines + 327):
                datafile[currentLine] = ""
                currentLine += 1
        else:
            currentLine += 1
    file.close()
    file2 = open('temp.md', 'w')
    file2.writelines(datafile)
    file2.close()

def nmapScan(ipList, outputFile, stealth):
    #nmap scan all active IPs
    #add nmap scans in md format to md report, create a section for each IP
    print("Nmap scan started")

    for ip in ipList:
        if stealth:
            cmd = "nmap -sS -P0 -T2 -oX temp.xml " + ip + " -Pn > /dev/null"
        else:
            cmd = "nmap -sV -sC -T4 -oX temp.xml " + ip + " -Pn > /dev/null"
        os.system(cmd)
        print("Nmap scan for " + ip + " complete")
        #convert to markdown and add to notes file
        cmd = "xsltproc temp.xml -o temp.md"
        os.system(cmd)
        cleanMDfile()
        notesFile = open(outputFile, "a")
        notesFile.write("\n
" + ip + "
\n")
        with open("temp.md") as f:
            for line in f:
                notesFile.write(line)
        listOfServices = getServiceListOutput()
        #remove temp file
        os.remove("temp.md")
    print("Nmap scan complete")
    return listOfServices

class ContentCallback:
    def __init__(self):
        self.contents = ''

    def content_callback(self, buf):
        self.contents = self.contents + str(buf)

def searchExploitDB(services, outputFile):
    #do exploit db search for all services and version numbers running on machines
    notesFile = open(outputFile, "a")
    notesFile.write("
\n
Exploits found:
")
    notesFile.close()
    for service in services:
        try:
            query = service + ' ' + 'site:https://www.exploit-db.com'
            for data in search(query, tld='com', num=10, start=0, stop=25, pause=2):
                if "https://www.exploit-db.com/exploits" in data:
                    t = ContentCallback()
                    curlObj = pycurl.Curl()
                    curlObj.setopt(curlObj.URL, '{}'.format(data))
                    curlObj.setopt(curlObj.WRITEFUNCTION, t.content_callback)
                    curlObj.perform()
                    curlObj.close()
                    #url = data
                    soup = BeautifulSoup(t.contents,'lxml')
                    desc = soup.find("meta", property="og:title")
                    publish = soup.find("meta", property="article:published_time")
                    #write results to md file
                    notesFile = open(outputFile, "a")
                    notesFile.write("
\n
" + desc.get('content') + "

\n")
                    notesFile.write("\n" + publish.get('content') + "
\n")
                    notesFile.write("\n" + data + "
\n")
                    notesFile.close()

        except Exception as e:
            print("Error connecting to ExploitDB")
            # print(e)

def SAMBAcheck(ip, outputFile):
    #check if samba is running on machine
    if ping_ip(ip):
        cmd = "nmap --script smb-enum-shares.nse,smb-os-discovery.nse,smb-vuln-conficker.nse,smb-vuln-cve2009-3103.nse,smb-vuln-cve-2017-7494.nse,smb-vuln-ms06-025.nse,smb-vuln-ms07-029.nse,smb-vuln-ms08-067.nse,smb-vuln-ms10-054.nse,smb-vuln-ms10-061.nse,smb-vuln-ms17-010.nse,smb-vuln-regsvc-dos.nse,smb-vuln-webexec.nse -p445 -oX temp.xml " + ip + " -Pn > /dev/null"
        os.system(cmd)
        #convert to markdown and add to notes file
        cmd = "xsltproc temp.xml -o temp.md"
        os.system(cmd)
        cleanMDfile()
        notesFile = open(outputFile, "a")
        with open("temp.md") as f:
            notesFile.write("\n
Samba enumeration:

\n")
            for line in f:
                notesFile.write(line)
        notesFile.close()
        #remove temp files
        os.remove("temp.xml")
        os.remove("temp.md")

def ftpCheck(ip, outputFile):
    #check if ftp is running on machine
    if ping_ip(ip):
        cmd = "nmap --script ftp-anon.nse,ftp-banner.nse,ftp-proftpd-backdoor.nse,ftp-vsftpd-backdoor.nse -p21 -oX temp.xml " + ip + " -Pn > /dev/null"
        os.system(cmd)
        #convert to markdown and add to notes file
        cmd = "xsltproc temp.xml -o temp.md"
        os.system(cmd)
        cleanMDfile()
        notesFile = open(outputFile, "a")
        with open("temp.md") as f:
            notesFile.write("\n
FTP enumeration:

\n")
            for line in f:
                notesFile.write(line)
        notesFile.close()
        #remove temp files
        os.remove("temp.xml")
        os.remove("temp.md")

def httpCheck(ip, ports, outputFile):
    #check if http is running on machine
    if ping_ip(ip):
        for port in ports:
            if port == "80" or port == "443" or port == "8080":
                cmd = "nikto -o temp.xml -Format xml --host http://" + ip + " -p " + port + " > /dev/null"
                os.system(cmd)
                #convert to markdown and add to notes file
                notesFile = open("temp.md", "w")
                notesFile.write("\n
HTTP enumeration:

\n")
                with open("temp.xml") as f:
                    for line in f:
                        notesFile.write(line)
                notesFile.close()
                cleanMDfile()
                notesFile = open(outputFile, "a")
                with open("temp.md") as f:
                    notesFile.write("\n")
                    for line in f:
                        notesFile.write(line)
                notesFile.close()
                #remove temp files
                os.remove("temp.xml")
                os.remove("temp.md")

def sshCheck(ip, outputFile):
    #check if ssh is running on machine
    if ping_ip(ip):
        cmd = "nmap --script ssh2-enum-algos.nse,ssh-hostkey.nse -p22 -oX temp.xml " + ip + " -Pn > /dev/null"
        os.system(cmd)
        #convert to markdown and add to notes file
        cmd = "xsltproc temp.xml -o temp.md"
        os.system(cmd)
        cleanMDfile()
        notesFile = open(outputFile, "a")
        with open("temp.md") as f:
            notesFile.write("\n
SSH enumeration:

\n")
            for line in f:
                notesFile.write(line)
        notesFile.close()
        #remove temp files
        os.remove("temp.xml")
        os.remove("temp.md")

def telnetCheck(ip, outputFile):
    #check if telnet is running on machine
    if ping_ip(ip):
        cmd = "nmap --script telnet-encryption.nse -p23 -oX temp.xml " + ip + " -Pn > /dev/null"
        os.system(cmd)
        #convert to markdown and add to notes file
        cmd = "xsltproc temp.xml -o temp.md"
        os.system(cmd)
        cleanMDfile()
        notesFile = open(outputFile, "a")
        with open("temp.md") as f:
            notesFile.write("\n
Telnet enumeration:

\n")
            for line in f:
                notesFile.write(line)
        notesFile.close()
        #remove temp files
        os.remove("temp.xml")
        os.remove("temp.md")

def snmpCheck(ip, outputFile):
    #check if snmp is running on machine
    if ping_ip(ip):
        cmd = "nmap --script snmp-netstat.nse,snmp-processes.nse,snmp-sysdescr.nse,snmp-win32-services.nse -p161 -oX temp.xml " + ip + " -Pn > /dev/null"
        os.system(cmd)
        #convert to markdown and add to notes file
        cmd = "xsltproc temp.xml -o temp.md"
        os.system(cmd)
        cleanMDfile()
        notesFile = open(outputFile, "a")
        with open("temp.md") as f:
            notesFile.write("\n
SNMP enumeration:

\n")
            for line in f:
                notesFile.write(line)
        notesFile.close()
        #remove temp files
        os.remove("temp.xml")
        os.remove("temp.md")

def mysqlCheck(ip, outputFile):
    #check if mysql is running on machine
    if ping_ip(ip):
        cmd = "nmap --script mysql-enum.nse -p3306 -oX temp.xml " + ip + " -Pn > /dev/null"
        os.system(cmd)
        #convert to markdown and add to notes file
        cmd = "xsltproc temp.xml -o temp.md"
        os.system(cmd)
        cleanMDfile()
        notesFile = open(outputFile, "a")
        with open("temp.md") as f:
            notesFile.write("\n
MySQL enumeration:

\n")
            for line in f:
                notesFile.write(line)
        notesFile.close()
        #remove temp files
        os.remove("temp.xml")
        os.remove("temp.md")

def icmpCheck(ip, outputFile):
    #check if icmp is running on machine
    if ping_ip(ip):
        cmd = "nmap --script icmp-echo.nse -p icmp -oX temp.xml " + ip + " -Pn > /dev/null"
        os.system(cmd)
        #convert to markdown and add to notes file
        cmd = "xsltproc temp.xml -o temp.md"
        os.system(cmd)
        cleanMDfile()
        notesFile = open(outputFile, "a")
        with open("temp.md") as f:
            notesFile.write("\n
ICMP enumeration:

\n")
            for line in f:
                notesFile.write(line)
        notesFile.close()
        #remove temp files
        os.remove("temp.xml")
        os.remove("temp.md")

def smtpCheck(ip, outputFile):
    #check if smtp is running on machine
    if ping_ip(ip):
        cmd = "nmap --script smtp-commands.nse,smtp-enum-users.nse,smtp-open-relay.nse -p25 -oX temp.xml " + ip + " -Pn > /dev/null"
        os.system(cmd)
        #convert to markdown and add to notes file
        cmd = "xsltproc temp.xml -o temp.md"
        os.system(cmd)
        cleanMDfile()
        notesFile = open(outputFile, "a")
        with open("temp.md") as f:
            notesFile.write("\n
SMTP enumeration:

\n")
            for line in f:
                notesFile.write(line)
        notesFile.close()
        #remove temp files
        os.remove("temp.xml")
        os.remove("temp.md")

def dnsCheck(ip, outputFile):
    #check if dns is running on machine
    if ping_ip(ip):
        cmd = "nmap --script dns-recursion.nse,dns-zone-transfer.nse -p53 -oX temp.xml " + ip + " -Pn > /dev/null"
        os.system(cmd)
        #convert to markdown and add to notes file
        cmd = "xsltproc temp.xml -o temp.md"
        os.system(cmd)
        cleanMDfile()
        notesFile = open(outputFile, "a")
        with open("temp.md") as f:
            notesFile.write("\n
DNS enumeration:

\n")
            for line in f:
                notesFile.write(line)
        notesFile.close()
        #remove temp files
        os.remove("temp.xml")
        os.remove("temp.md")

def pop3Check(ip, outputFile):
    #check if pop3 is running on machine
    if ping_ip(ip):
        cmd = "nmap --script pop3-capabilities.nse,pop3-enum-users.nse -p110 -oX temp.xml " + ip + " -Pn > /dev/null"
        os.system(cmd)
        #convert to markdown and add to notes file
        cmd = "xsltproc temp.xml -o temp.md"
        os.system(cmd)
        cleanMDfile()
        notesFile = open(outputFile, "a")
        with open("temp.md") as f:
            notesFile.write("\n
POP3 enumeration:

\n")
            for line in f:
                notesFile.write(line)
        notesFile.close()
        #remove temp files
        os.remove("temp.xml")
        os.remove("temp.md")

main()


import sklearn
import sklearn.cross_validation
import sklearn.preprocessing
import numpy

class BalancedShuffleSplit(object):
	"""docstring for BalancedShuffleSplit"""
	def __init__(self, y, n = 5, train_size = 0.25, random_state = numpy.random.RandomState()):
		super(BalancedShuffleSplit, self).__init__()
		self._nsamples = len(y)
		self._n = n
		self._rng = random_state
		if 0 < train_size < 1:
			self._trainsize = int(train_size*self._nsamples)
		else:
			self._trainsize = train_size

		self._labelencoder = sklearn.preprocessing.LabelEncoder()
		self._y = self._labelencoder.fit_transform(y)#make the labels in range [0,c), where c is number of labels
		self._nclasses = max(self._y)+1
		self._buckets = [numpy.where(self._y==i)[0] for i in numpy.unique(self._y)]

		self._samplesperclass = self._trainsize/self._nclasses
		self._excess = self._trainsize%self._nclasses

		classes_not_ok = []
		for i in numpy.unique(self._y):

			if not (sum(self._y==i)>(self._samplesperclass+self._excess)):
				classes_not_ok.append(i)

		#print self._trainsize
		#print self._samplesperclass+self._excess

		if len(classes_not_ok)>0:
			raise Exception('Need at least %d samples for classes = %s'%(self._samplesperclass+self._excess+1, self._labelencoder.inverse_transform(classes_not_ok)))

	def __iter__(self):
		for i in xrange(self._n):
			yield self._generateBalancedShuffleSplit()

	def _generateBalancedShuffleSplit(self):
		train = numpy.ones(0)
		test = numpy.ones(0)
		idx = 0
		for i in self._buckets:
			self._rng.shuffle(i)
			if idx<self._excess:
				trainsize = self._samplesperclass+1
			else:
				trainsize = self._samplesperclass
			#print trainsize
			train = numpy.append(train, i[:trainsize])
			test = numpy.append(test, i[trainsize:])
			idx+=1

		# print (train, test)
		return (train.astype(int), test.astype(int))


from collections import OrderedDict
from typing import Any, Dict, List, Optional, Type, Set, cast, Tuple

from phidata.k8s.args import K8sArgs
from phidata.k8s.api_client import K8sApiClient
from phidata.k8s.resource.base import K8sResource
from phidata.k8s.create.group import CreateK8sResourceGroup
from phidata.k8s.resource.group import (
    K8sResourceGroup,
    K8sBuildContext,
)
from phidata.k8s.resource.utils import (
    get_k8s_resources_from_group,
    filter_and_flatten_k8s_resource_groups,
)
from phidata.utils.cli_console import (
    print_info,
    print_heading,
    print_subheading,
    confirm_yes_no,
)
from phidata.utils.log import logger

class K8sWorker:
    """This class interacts with the Kubernetes API."""

    def __init__(self, k8s_args: K8sArgs) -> None:
        self.k8s_args: K8sArgs = k8s_args
        self.k8s_api_client: K8sApiClient = K8sApiClient(
            context=self.k8s_args.context,
            kubeconfig_path=self.k8s_args.kubeconfig_path,
        )
        logger.debug(f"**-+-** K8sWorker created")

    def is_client_initialized(self) -> bool:
        return self.k8s_api_client.is_initialized()

    def are_resources_active(self) -> bool:
        # TODO: fix this
        return False

    ######################################################
    ## Build Resources
    ######################################################

    def build_k8s_resource_groups(
        self,
        app_filter: Optional[str] = None,
    ) -> Optional[Dict[str, K8sResourceGroup]]:
        """
        Build the K8sResourceGroups for the requested apps

        Step 1: Convert each PhidataApp to K8sResourceGroups.
        Step 2: Convert each CreateK8sResourceGroup to K8sResourceGroups.
        Step 3: Add all K8sResourceGroups to the self.k8s_resources dict
        """
        logger.debug("-*- Initializing K8sResourceGroups")

        k8s_resource_groups: Optional[Dict[str, K8sResourceGroup]] = None

        # track the total number of K8sResourceGroups to build for validation
        apps_to_build = self.k8s_args.apps
        resources_to_build = self.k8s_args.resources
        create_resources_to_build = self.k8s_args.create_resources

        num_apps = len(apps_to_build) if apps_to_build is not None else 0
        num_resources = len(resources_to_build) if resources_to_build is not None else 0
        num_create_resources = (
            len(create_resources_to_build)
            if create_resources_to_build is not None
            else 0
        )
        num_rgs_to_build = num_apps + num_resources + num_create_resources
        num_rgs_built = 0

        # Step 1: Convert each PhidataApp to K8sResourceGroups.
        if apps_to_build is not None and isinstance(apps_to_build, list):
            for app in apps_to_build:
                if app.args is None:
                    logger.error("Args for App {} are None".format(app))
                    num_rgs_built += 1
                    continue

                if not app.enabled:
                    logger.debug(f"{app.name} disabled")
                    num_rgs_built += 1
                    continue

                # skip apps not matching app_filter if provided
                if app_filter is not None:
                    if app_filter.lower() not in app.name:
                        logger.debug(f"Skipping {app.name}")
                        num_rgs_built += 1
                        continue

                logger.debug("-*- App: {}".format(app.name))

                ######################################################################
                # NOTE: VERY IMPORTANT TO GET RIGHT
                # Pass down parameters from K8sArgs -> PhidataApp
                # The K8sConfig inherits these params from the WorkspaceConfig
                # 1. Pass down the paths from the WorkspaceConfig
                # 2. Pass down k8s_env
                # 3. Pass down common cloud configuration. eg: aws_region, aws_profile
                # This should match phidata.infra.prep_infra_config.prep_infra_config()
                ######################################################################

                # -*- Path parameters
                app.scripts_dir = self.k8s_args.scripts_dir
                app.storage_dir = self.k8s_args.storage_dir
                app.meta_dir = self.k8s_args.meta_dir
                app.products_dir = self.k8s_args.products_dir
                app.notebooks_dir = self.k8s_args.notebooks_dir
                app.workflows_dir = self.k8s_args.workflows_dir
                # The ws_root_path is the ROOT directory for the workspace
                app.workspace_root_path = self.k8s_args.workspace_root_path
                app.workspace_config_dir = self.k8s_args.workspace_config_dir
                app.workspace_config_file_path = (
                    self.k8s_args.workspace_config_file_path
                )

                # -*- Environment parameters
                # only update the params if they are not available on the app.
                # so we can prefer the app param if provided
                if app.k8s_env is None and self.k8s_args.k8s_env is not None:
                    app.k8s_env = self.k8s_args.k8s_env

                # -*- AWS parameters
                # only update the params if they are not available on the app.
                # so we can prefer the app param if provided
                if app.aws_region is None and self.k8s_args.aws_region is not None:
                    app.aws_region = self.k8s_args.aws_region
                if app.aws_profile is None and self.k8s_args.aws_profile is not None:
                    app.aws_profile = self.k8s_args.aws_profile
                if (
                    app.aws_config_file is None
                    and self.k8s_args.aws_config_file is not None
                ):
                    app.aws_config_file = self.k8s_args.aws_config_file
                if (
                    app.aws_shared_credentials_file is None
                    and self.k8s_args.aws_shared_credentials_file is not None
                ):
                    app.aws_shared_credentials_file = (
                        self.k8s_args.aws_shared_credentials_file
                    )

                app_rgs: Optional[
                    Dict[str, K8sResourceGroup]
                ] = app.get_k8s_resource_groups(
                    k8s_build_context=K8sBuildContext(
                        namespace=self.k8s_args.namespace,
                        context=self.k8s_args.context,
                        service_account_name=self.k8s_args.service_account_name,
                        labels=self.k8s_args.common_labels,
                    )
                )
                if app_rgs is not None:
                    if k8s_resource_groups is None:
                        k8s_resource_groups = OrderedDict()
                    k8s_resource_groups.update(app_rgs)
                    num_rgs_built += 1

        # Step 2: Convert each CreateK8sResourceGroup to K8sResourceGroups.
        if create_resources_to_build is not None and isinstance(
            create_resources_to_build, list
        ):
            for create_resource in create_resources_to_build:
                if not create_resource.enabled:
                    logger.debug(f"{create_resource.name} disabled")
                    num_rgs_built += 1
                    continue

                # skip groups not matching app_filter if provided
                if app_filter is not None:
                    if app_filter.lower() not in create_resource.name:
                        logger.debug(f"Skipping {create_resource.name}")
                        num_rgs_built += 1
                        continue

                logger.debug("-*- Resource: {}".format(create_resource.name))

                if isinstance(create_resource, CreateK8sResourceGroup):
                    # Create the K8sResourceGroup
                    k8s_resource_group: Optional[
                        K8sResourceGroup
                    ] = create_resource.create()
                    if k8s_resource_group is not None:
                        if k8s_resource_groups is None:
                            k8s_resource_groups = OrderedDict()
                        k8s_resource_groups[
                            k8s_resource_group.name
                        ] = k8s_resource_group
                    num_rgs_built += 1

        # Step 3: Add all K8sResourceGroups to the k8s_resource_groups dict
        if resources_to_build is not None and isinstance(resources_to_build, list):
            for resource in resources_to_build:
                if not resource.enabled:
                    logger.debug(f"{resource.name} disabled")
                    num_rgs_built += 1
                    continue

                # skip groups not matching app_filter if provided
                if app_filter is not None:
                    if app_filter.lower() not in resource.name:
                        logger.debug(f"Skipping {resource.name}")
                        num_rgs_built += 1
                        continue

                logger.debug("-*- Resource: {}".format(resource.name))

                if isinstance(resource, K8sResourceGroup):
                    if k8s_resource_groups is None:
                        k8s_resource_groups = OrderedDict()
                    k8s_resource_groups[resource.name] = resource
                    num_rgs_built += 1

        logger.debug(f"# K8sResourceGroups built: {num_rgs_built}/{num_rgs_to_build}")
        return k8s_resource_groups

    ######################################################
    ## Create Resources
    ######################################################

    def create_resources(
        self,
        name_filter: Optional[str] = None,
        type_filter: Optional[str] = None,
        app_filter: Optional[str] = None,
        auto_confirm: Optional[bool] = False,
    ) -> bool:
        logger.debug("-*- Creating K8sResources")

        k8s_resource_groups: Optional[
            Dict[str, K8sResourceGroup]
        ] = self.build_k8s_resource_groups(app_filter=app_filter)

        if k8s_resource_groups is None:
            print_info("No resources available")
            return True

        # Create a list of tuples with 3 parts
        #   1. Resource group name
        #   2. Resource group weight
        #   3. List of Resources in group after filters
        k8s_resources_to_create: List[Tuple[str, int, List[K8sResource]]] = []
        for k8s_rg_name, k8s_rg in k8s_resource_groups.items():
            logger.debug(f"Processing: {k8s_rg_name}")

            k8s_resources_in_group = get_k8s_resources_from_group(
                k8s_resource_group=k8s_rg,
                name_filter=name_filter,
                type_filter=type_filter,
            )
            num_resources_in_group = len(k8s_resources_in_group)
            logger.debug(f"Matched {num_resources_in_group} resources")
            if num_resources_in_group > 0:
                k8s_resources_to_create.append(
                    (k8s_rg_name, k8s_rg.weight, k8s_resources_in_group)
                )

        if len(k8s_resources_to_create) == 0:
            print_subheading("No K8sResources to create")
            return True

        k8s_resources_to_create_sorted: List[
            Tuple[str, int, List[K8sResource]]
        ] = sorted(
            k8s_resources_to_create,
            key=lambda x: x[1],
        )

        # Validate resources to be created
        group_number = 1
        resource_number = 0
        if not auto_confirm:
            print_heading("--**-- Confirm resources:")
            for (
                group_name,
                group_weight,
                resource_list,
            ) in k8s_resources_to_create_sorted:
                print_subheading(f"\n{group_number}. {group_name}")
                resource_number += len(resource_list)
                group_number += 1
                for resource in resource_list:
                    print_info(
                        f"  -+-> {resource.get_resource_type()}: {resource.get_resource_name()}"
                    )
            print_info(f"\nNamespace: {self.k8s_args.namespace}")
            print_info(f"Context: {self.k8s_args.context}")
            print_info(f"\nTotal {resource_number} resources")
            confirm = confirm_yes_no("\nConfirm deploy")
            if not confirm:
                print_info("-*-")
                print_info("-*- Skipping deploy")
                print_info("-*-")
                exit(0)

        # track the total number of K8sResources to create for validation
        num_resources_to_create: int = 0
        num_resources_created: int = 0

        for (group_name, group_weight, resource_list) in k8s_resources_to_create_sorted:
            print_subheading(f"\n-*- {group_name}")
            num_resources_to_create += len(resource_list)
            for resource in resource_list:
                if resource:
                    print_info(
                        f"\n-==+==- {resource.get_resource_type()}: {resource.get_resource_name()}"
                    )
                    logger.debug(f"Resource: {resource}")
                    try:
                        _resource_created = resource.create(
                            k8s_client=self.k8s_api_client
                        )
                        if _resource_created:
                            num_resources_created += 1
                            logger.info("Resource created")
                            logger.debug(
                                f"continue_on_create_failure: {self.k8s_args.continue_on_create_failure}"
                            )
                        else:
                            logger.error(
                                f"Resource {resource.get_resource_type()}: {resource.get_resource_name()} could not be created."
                            )
                            if self.k8s_args.continue_on_create_failure is False:
                                return False
                    except Exception as e:
                        logger.error(
                            f"-==+==--> Resource {resource.get_resource_type()}: {resource.get_resource_name()} could not be created."
                        )
                        logger.error("Error: {}".format(e))
                        logger.error(
                            "Skipping resource creation, please fix and try again..."
                        )

        print_info(
            f"\n# Resources created: {num_resources_created}/{num_resources_to_create}"
        )
        if num_resources_to_create == num_resources_created:
            return True

        logger.error(
            f"Resources created: {num_resources_created} do not match resources required: {num_resources_to_create}"
        )
        return False

    def create_resources_dry_run(
        self,
        name_filter: Optional[str] = None,
        type_filter: Optional[str] = None,
        app_filter: Optional[str] = None,
        auto_confirm: Optional[bool] = False,
    ) -> None:
        logger.debug("-*- Creating K8sResources")

        k8s_resource_groups: Optional[
            Dict[str, K8sResourceGroup]
        ] = self.build_k8s_resource_groups(app_filter=app_filter)

        if k8s_resource_groups is None:
            print_info("No resources available")
            return

        # Create a list of tuples with 3 parts
        #   1. Resource group name
        #   2. Resource group weight
        #   3. List of Resources in group after filters
        k8s_resources_to_create: List[Tuple[str, int, List[K8sResource]]] = []
        for k8s_rg_name, k8s_rg in k8s_resource_groups.items():
            logger.debug(f"Processing: {k8s_rg_name}")

            k8s_resources_in_group = get_k8s_resources_from_group(
                k8s_resource_group=k8s_rg,
                name_filter=name_filter,
                type_filter=type_filter,
            )
            num_resources_in_group = len(k8s_resources_in_group)
            logger.debug(f"Matched {num_resources_in_group} resources")
            if num_resources_in_group > 0:
                k8s_resources_to_create.append(
                    (k8s_rg_name, k8s_rg.weight, k8s_resources_in_group)
                )

        if len(k8s_resources_to_create) == 0:
            print_info("No K8sResources to create")
            return

        k8s_resources_to_create_sorted: List[
            Tuple[str, int, List[K8sResource]]
        ] = sorted(
            k8s_resources_to_create,
            key=lambda x: x[1],
        )

        group_number = 1
        num_resources_to_create: int = 0
        print_heading("--**-- Kubernetes resources:")
        for (group_name, group_weight, resource_list) in k8s_resources_to_create_sorted:
            print_subheading(f"\n{group_number}. {group_name}")
            num_resources_to_create += len(resource_list)
            group_number += 1
            for resource in resource_list:
                print_info(
                    f"  -+-> {resource.get_resource_type()}: {resource.get_resource_name()}"
                )
        print_info(f"\nNamespace: {self.k8s_args.namespace}")
        print_info(f"Context: {self.k8s_args.context}")
        print_info(f"\nTotal {num_resources_to_create} resources")

    ######################################################
    ## Get Resources
    ######################################################

    def get_resources(
        self,
        name_filter: Optional[str] = None,
        type_filter: Optional[str] = None,
        app_filter: Optional[str] = None,
    ) -> Optional[List[K8sResource]]:

        logger.debug("-*- Getting K8sResources")

        k8s_resource_groups: Optional[
            Dict[str, K8sResourceGroup]
        ] = self.build_k8s_resource_groups(app_filter=app_filter)

        if k8s_resource_groups is None:
            print_info("No resources available")
            return None

        k8s_resources: Optional[
            List[K8sResource]
        ] = filter_and_flatten_k8s_resource_groups(
            k8s_resource_groups=k8s_resource_groups,
            name_filter=name_filter,
            type_filter=type_filter,
            app_filter=app_filter,
        )

        return k8s_resources

    ######################################################
    ## Delete Resources
    ######################################################

    def delete_resources(
        self,
        name_filter: Optional[str] = None,
        type_filter: Optional[str] = None,
        app_filter: Optional[str] = None,
        auto_confirm: Optional[bool] = False,
    ) -> bool:
        logger.debug("-*- Deleting K8sResources")

        k8s_resource_groups: Optional[
            Dict[str, K8sResourceGroup]
        ] = self.build_k8s_resource_groups(app_filter=app_filter)

        if k8s_resource_groups is None:
            print_info("No resources available")
            return True

        # Create a list of tuples with 3 parts
        #   1. Resource group name
        #   2. Resource group weight
        #   3. List of Resources in group after filters
        k8s_resources_to_delete: List[Tuple[str, int, List[K8sResource]]] = []
        for k8s_rg_name, k8s_rg in k8s_resource_groups.items():
            logger.debug(f"Processing: {k8s_rg_name}")

            k8s_resources_in_group = get_k8s_resources_from_group(
                k8s_resource_group=k8s_rg,
                name_filter=name_filter,
                type_filter=type_filter,
                sort_order="delete",
            )
            num_resources_in_group = len(k8s_resources_in_group)
            logger.debug(f"Matched {num_resources_in_group} resources")
            if num_resources_in_group > 0:
                k8s_resources_to_delete.append(
                    (k8s_rg_name, k8s_rg.weight, k8s_resources_in_group)
                )

        if len(k8s_resources_to_delete) == 0:
            print_subheading("No K8sResources to delete")
            return True

        k8s_resources_to_delete_sorted: List[
            Tuple[str, int, List[K8sResource]]
        ] = sorted(
            k8s_resources_to_delete,
            key=lambda x: x[1],
            reverse=True,
        )

        # Validate resources to be created
        group_number = 1
        resource_number = 0
        if not auto_confirm:
            print_heading("--**-- Confirm resources:")
            for (
                group_name,
                group_weight,
                resource_list,
            ) in k8s_resources_to_delete_sorted:
                print_subheading(f"\n{group_number}. {group_name}")
                resource_number += len(resource_list)
                group_number += 1
                for resource in resource_list:
                    print_info(
                        f"  -+-> {resource.get_resource_type()}: {resource.get_resource_name()}"
                    )
            print_info(f"\nNamespace: {self.k8s_args.namespace}")
            print_info(f"Context: {self.k8s_args.context}")
            print_info(f"\nTotal {resource_number} resources")
            confirm = confirm_yes_no("\nConfirm delete")
            if not confirm:
                print_info("-*-")
                print_info("-*- Skipping delete")
                print_info("-*-")
                exit(0)

        # track the total number of K8sResources to create for validation
        num_resources_to_delete: int = 0
        num_resources_deleted: int = 0

        for (group_name, group_weight, resource_list) in k8s_resources_to_delete_sorted:
            print_subheading(f"\n-*- {group_name}")
            num_resources_to_delete += len(resource_list)
            for resource in resource_list:
                if resource:
                    print_info(
                        f"\n-==+==- {resource.get_resource_type()}: {resource.get_resource_name()}"
                    )
                    # logger.debug(resource)
                    try:
                        _resource_deleted = resource.delete(
                            k8s_client=self.k8s_api_client
                        )
                        if _resource_deleted:
                            num_resources_deleted += 1
                            print_info("Resource deleted")
                        else:
                            logger.error(
                                f"Resource {resource.get_resource_type()}: {resource.get_resource_name()} could not be deleted."
                            )
                            if not self.k8s_args.continue_on_delete_failure:
                                return False
                    except Exception as e:
                        logger.error(
                            f"-==+==--> Resource {resource.get_resource_type()}: {resource.get_resource_name()} could not be created."
                        )
                        logger.error("Error: {}".format(e))
                        logger.error(
                            "Skipping resource creation, please fix and try again..."
                        )

        print_info(
            f"\n# Resources deleted: {num_resources_deleted}/{num_resources_to_delete}"
        )
        if num_resources_to_delete == num_resources_deleted:
            return True

        logger.error(
            f"Resources deleted: {num_resources_deleted} do not match resources required: {num_resources_to_delete}"
        )
        return False

    def delete_resources_dry_run(
        self,
        name_filter: Optional[str] = None,
        type_filter: Optional[str] = None,
        app_filter: Optional[str] = None,
        auto_confirm: Optional[bool] = False,
    ) -> None:
        logger.debug("-*- Deleting K8sResources")

        k8s_resource_groups: Optional[
            Dict[str, K8sResourceGroup]
        ] = self.build_k8s_resource_groups(app_filter=app_filter)

        if k8s_resource_groups is None:
            print_info("No resources available")
            return None

        # Create a list of tuples with 3 parts
        #   1. Resource group name
        #   2. Resource group weight
        #   3. List of Resources in group after filters
        k8s_resources_to_delete: List[Tuple[str, int, List[K8sResource]]] = []
        for k8s_rg_name, k8s_rg in k8s_resource_groups.items():
            logger.debug(f"Processing: {k8s_rg_name}")

            k8s_resources_in_group = get_k8s_resources_from_group(
                k8s_resource_group=k8s_rg,
                name_filter=name_filter,
                type_filter=type_filter,
                sort_order="delete",
            )
            num_resources_in_group = len(k8s_resources_in_group)
            logger.debug(f"Matched {num_resources_in_group} resources")
            if num_resources_in_group > 0:
                k8s_resources_to_delete.append(
                    (k8s_rg_name, k8s_rg.weight, k8s_resources_in_group)
                )

        if len(k8s_resources_to_delete) == 0:
            print_info("No K8sResources to create")
            return

        k8s_resources_to_delete_sorted: List[
            Tuple[str, int, List[K8sResource]]
        ] = sorted(
            k8s_resources_to_delete,
            key=lambda x: x[1],
            reverse=True,
        )

        group_number = 1
        num_resources_to_delete: int = 0
        print_heading("--**-- Kubernetes resources:")
        for (group_name, group_weight, resource_list) in k8s_resources_to_delete_sorted:
            print_subheading(f"\n{group_number}. {group_name}")
            num_resources_to_delete += len(resource_list)
            group_number += 1
            for resource in resource_list:
                print_info(
                    f"  -+-> {resource.get_resource_type()}: {resource.get_resource_name()}"
                )
        print_info(f"\nNamespace: {self.k8s_args.namespace}")
        print_info(f"Context: {self.k8s_args.context}")
        print_info(f"\nTotal {num_resources_to_delete} resources")

    ######################################################
    ## Patch Resources
    ######################################################

    def patch_resources(
        self,
        name_filter: Optional[str] = None,
        type_filter: Optional[str] = None,
        app_filter: Optional[str] = None,
        auto_confirm: Optional[bool] = False,
    ) -> bool:
        logger.debug("-*- Patching K8sResources")

        k8s_resource_groups: Optional[
            Dict[str, K8sResourceGroup]
        ] = self.build_k8s_resource_groups(app_filter=app_filter)

        if k8s_resource_groups is None:
            print_info("No resources available")
            return True

        # Create a list of tuples with 3 parts
        #   1. Resource group name
        #   2. Resource group weight
        #   3. List of Resources in group after filters
        k8s_resources_to_patch: List[Tuple[str, int, List[K8sResource]]] = []
        for k8s_rg_name, k8s_rg in k8s_resource_groups.items():
            logger.debug(f"Processing: {k8s_rg_name}")

            k8s_resources_in_group = get_k8s_resources_from_group(
                k8s_resource_group=k8s_rg,
                name_filter=name_filter,
                type_filter=type_filter,
            )
            num_resources_in_group = len(k8s_resources_in_group)
            logger.debug(f"Matched {num_resources_in_group} resources")
            if num_resources_in_group > 0:
                k8s_resources_to_patch.append(
                    (k8s_rg_name, k8s_rg.weight, k8s_resources_in_group)
                )

        if len(k8s_resources_to_patch) == 0:
            print_subheading("No K8sResources to patch")
            return True

        k8s_resources_to_patch_sorted: List[
            Tuple[str, int, List[K8sResource]]
        ] = sorted(
            k8s_resources_to_patch,
            key=lambda x: x[1],
        )

        # Validate resources to be patched
        group_number = 1
        resource_number = 0
        if not auto_confirm:
            print_heading("--**-- Confirm resources:")
            for (
                group_name,
                group_weight,
                resource_list,
            ) in k8s_resources_to_patch_sorted:
                print_subheading(f"\n{group_number}. {group_name}")
                resource_number += len(resource_list)
                group_number += 1
                for resource in resource_list:
                    print_info(
                        f"  -+-> {resource.get_resource_type()}: {resource.get_resource_name()}"
                    )
            print_info(f"\nNamespace: {self.k8s_args.namespace}")
            print_info(f"Context: {self.k8s_args.context}")
            print_info(f"\nTotal {resource_number} resources")
            confirm = confirm_yes_no("\nConfirm patch")
            if not confirm:
                print_info("-*-")
                print_info("-*- Skipping patch")
                print_info("-*-")
                exit(0)

        # track the total number of K8sResources to patch for validation
        num_resources_to_patch: int = 0
        num_resources_patched: int = 0

        for (group_name, group_weight, resource_list) in k8s_resources_to_patch_sorted:
            print_subheading(f"\n-*- {group_name}")
            num_resources_to_patch += len(resource_list)
            for resource in resource_list:
                if resource:
                    print_info(
                        f"\n-==+==- {resource.get_resource_type()}: {resource.get_resource_name()}"
                    )
                    # logger.debug(resource)
                    try:
                        _resource_patched = resource.update(
                            k8s_client=self.k8s_api_client
                        )
                        if _resource_patched:
                            num_resources_patched += 1
                            print_info("Resource patched")
                        else:
                            logger.error(
                                f"Resource {resource.get_resource_type()}: {resource.get_resource_name()} could not be patched."
                            )
                            if not self.k8s_args.continue_on_patch_failure:
                                return False
                    except Exception as e:
                        logger.error(
                            f"-==+==--> Resource {resource.get_resource_type()}: {resource.get_resource_name()} could not be patched."
                        )
                        logger.error("Error: {}".format(e))
                        logger.error(
                            "Skipping resource creation, please fix and try again..."
                        )

        print_info(
            f"\n# Resources patched: {num_resources_patched}/{num_resources_to_patch}"
        )
        if num_resources_to_patch == num_resources_patched:
            return True

        logger.error(
            f"Resources patched: {num_resources_patched} do not match resources required: {num_resources_to_patch}"
        )
        return False

    def patch_resources_dry_run(
        self,
        name_filter: Optional[str] = None,
        type_filter: Optional[str] = None,
        app_filter: Optional[str] = None,
        auto_confirm: Optional[bool] = False,
    ) -> None:
        logger.debug("-*- Patching K8sResources")

        k8s_resource_groups: Optional[
            Dict[str, K8sResourceGroup]
        ] = self.build_k8s_resource_groups(app_filter=app_filter)

        if k8s_resource_groups is None:
            print_info("No resources available")
            return None

        # Create a list of tuples with 3 parts
        #   1. Resource group name
        #   2. Resource group weight
        #   3. List of Resources in group after filters
        k8s_resources_to_patch: List[Tuple[str, int, List[K8sResource]]] = []
        for k8s_rg_name, k8s_rg in k8s_resource_groups.items():
            logger.debug(f"Processing: {k8s_rg_name}")

            k8s_resources_in_group = get_k8s_resources_from_group(
                k8s_resource_group=k8s_rg,
                name_filter=name_filter,
                type_filter=type_filter,
            )
            num_resources_in_group = len(k8s_resources_in_group)
            logger.debug(f"Matched {num_resources_in_group} resources")
            if num_resources_in_group > 0:
                k8s_resources_to_patch.append(
                    (k8s_rg_name, k8s_rg.weight, k8s_resources_in_group)
                )

        if len(k8s_resources_to_patch) == 0:
            print_info("No K8sResources to patch")
            return

        k8s_resources_to_patch_sorted: List[
            Tuple[str, int, List[K8sResource]]
        ] = sorted(
            k8s_resources_to_patch,
            key=lambda x: x[1],
        )

        group_number = 1
        num_resources_to_patch: int = 0
        print_heading("--**-- Kubernetes resources:")
        for (group_name, group_weight, resource_list) in k8s_resources_to_patch_sorted:
            print_subheading(f"\n{group_number}. {group_name}")
            num_resources_to_patch += len(resource_list)
            group_number += 1
            for resource in resource_list:
                print_info(
                    f"  -+-> {resource.get_resource_type()}: {resource.get_resource_name()}"
                )
        print_info(f"\nNamespace: {self.k8s_args.namespace}")
        print_info(f"Context: {self.k8s_args.context}")
        print_info(f"\nTotal {num_resources_to_patch} resources")

    ######################################################
    ## End
    ######################################################

    ######################################################
    ## List Resources
    ######################################################

    # def get_active_resource_classes(
    #     self,
    #     type_filters: Optional[List[str]] = None,
    #     name_filters: Optional[List[str]] = None,
    # ) -> Optional[Set[Type[K8sResource]]]:
    #     """This function scans all K8sResourceGroups under this client and
    #     returns a set of different types of K8sResource(s) as a set of classes.
    #
    #     When do we need this:
    #     Let's say your app has 1 Deployment, 1 Pod and 1 Service. You deploy it and now want
    #     to see the different types of objects running on your cluster on your cluster.
    #     To do that you need to query the API for each and every resource type to see if there are any running objects of that type.
    #     We use this function to filter & dedup the resource types this K8s client is managing.
    #     This way we don't end up scanning everything.
    #
    #     Also super useful when you want to just see Deployments running, faster responses.
    #     """
    #     if self.k8s_resources is None:
    #         logger.debug("No K8sResourceGroups available")
    #         return None
    #
    #     _filtered_k8s_resources: Optional[
    #         List[K8sResource]
    #     ] = filter_and_flatten_k8s_resource_groups(
    #         k8s_resource_groups=self.k8s_resources,
    #         type_filters=type_filters,
    #         name_filters=name_filters,
    #         sort_order="create",
    #     )
    #     return dedup_resource_types(_filtered_k8s_resources)
    #
    # def get_active_resources(
    #     self,
    #     type_filters: Optional[List[str]] = None,
    #     name_filters: Optional[List[str]] = None,
    # ) -> Optional[Dict[str, List]]:
    #     """Reads the K8s Cluster and returns all active k8s_resources which satisfy the filters."""
    #
    #     from kubernetes.client.rest import ApiException
    #
    #     active_resource_classes: Optional[
    #         Set[Type[K8sResource]]
    #     ] = self.get_active_resource_classes(type_filters, name_filters)
    #
    #     if active_resource_classes is None:
    #         return None
    #
    #     _active_ns = self.get_k8s_namespace_to_use()
    #     active_k8s_objects: Dict[str, List] = defaultdict(list)
    #     for resource_class in active_resource_classes:
    #         resource_type: str = resource_class.__name__
    #         logger.debug(f"Resource Type: {resource_type}")
    #         try:
    #             _active_objects: Optional[List[Any]] = resource_class.get_from_cluster(
    #                 k8s_client=cast(K8sApiClient, self.k8s_api_client), namespace=_active_ns
    #             )
    #             if _active_objects is not None and isinstance(_active_objects, list):
    #                 active_k8s_objects[resource_type] = _active_objects
    #         except ApiException as e:
    #             logger.debug(
    #                 f"ApiException while getting {resource_type}, reason: {e.reason}"
    #             )
    #
    #     return active_k8s_objects
    #
    # def print(
    #     self,
    #     type_filters: Optional[List[str]] = None,
    #     name_filters: Optional[List[str]] = None,
    #     as_yaml: Optional[bool] = True,
    # ) -> None:
    #
    #     active_k8s_objects: Optional[Dict[str, List]] = self.get_active_resources(
    #         type_filters, name_filters
    #     )
    #     if active_k8s_objects:
    #         for _r_type, active_resources in active_k8s_objects.items():
    #             print("\n{}:".format(_r_type))
    #             for _r in active_resources:
    #                 print("\t- {}".format(_r))

    ######################################################
    ## Debug
    ######################################################

    # def debug(
    #     self,
    #     type_filters: Optional[List[str]] = None,
    #     name_filters: Optional[List[str]] = None,
    # ) -> None:
    #
    #     if self._k8s_client is None:
    #         logger.debug("No K8sApiClient available")
    #         return
    #     if self.k8s_resources is None:
    #         logger.debug("No K8sResourceGroups available")
    #         return
    #     _filtered_k8s_resources: Optional[
    #         List[K8sResource]
    #     ] = filter_and_flatten_k8s_resource_groups(
    #         k8s_resource_groups=self.k8s_resources,
    #         type_filters=type_filters,
    #         name_filters=name_filters,
    #         sort_order="create",
    #     )
    #     if _filtered_k8s_resources:
    #         for resource in _filtered_k8s_resources:
    #             logger.debug(f"Resource: {resource.metadata.name}")
    #             if resource and self.k8s_api_client:
    #                 resource.debug()

    ######################################################
    ## K8sApiClient
    ######################################################

    # def get_k8s_namespace_to_use(self) -> str:
    #     if self.k8s_resources is None:
    #         return DEFAULT_K8S_NAMESPACE
    #     ns_rg: K8sResourceGroup = self.k8s_resources[
    #         NAMESPACE_RESOURCE_GROUP_KEY
    #     ]
    #     if ns_rg and ns_rg.enabled and ns_rg.ns is not None:
    #         return ns_rg.ns.get_resource_name()
    #     return DEFAULT_K8S_NAMESPACE
    #
    # def get_k8s_service_account_to_use(self) -> str:
    #     if self.k8s_resources is None:
    #         return DEFAULT_K8S_SERVICE_ACCOUNT
    #     rbac_rg: K8sResourceGroup = self.k8s_resources[RBAC_RESOURCE_GROUP_KEY]
    #     if rbac_rg and rbac_rg.enabled and rbac_rg.sa is not None:
    #         return rbac_rg.sa.get_resource_name()
    #     return DEFAULT_K8S_SERVICE_ACCOUNT

    ######################################################
    ## Describe Resources
    ######################################################

    # def print_k8s_resources(
    #     self,
    #     type_filters: Optional[List[str]] = None,
    #     name_filters: Optional[List[str]] = None,
    #     op_format: str = "yaml",  # yaml or json
    # ) -> None:
    #
    #     if op_format not in ("yaml", "json"):
    #         print(f"Output format {op_format} not supported")
    #         return
    #
    #     if self.k8s_resources is None:
    #         logger.debug("No K8sResourceGroups available")
    #         return
    #
    #     self.print_k8s_resource_groups(
    #         k8s_resource_groups=self.k8s_resources,
    #         type_filters=type_filters,
    #         name_filters=name_filters,
    #         as_yaml=(op_format == "yaml"),
    #     )
    #
    # @staticmethod
    # def print_k8s_resource_groups(
    #     k8s_resource_groups: Dict[str, K8sResourceGroup],
    #     type_filters: Optional[List[str]] = None,
    #     name_filters: Optional[List[str]] = None,
    #     as_yaml: Optional[bool] = True,
    # ) -> None:
    #
    #     print(
    #         "Resources: {}/{}".format(
    #             type_filters if type_filters else "*",
    #             name_filters if name_filters else "*",
    #         )
    #     )
    #     _filtered_k8s_resources: Optional[
    #         List[K8sResource]
    #     ] = filter_and_flatten_k8s_resource_groups(
    #         k8s_resource_groups=k8s_resource_groups,
    #         type_filters=type_filters,
    #         name_filters=name_filters,
    #         sort_order="create",
    #     )
    #     _resources: List[str] = []
    #     if _filtered_k8s_resources:
    #         for resource in _filtered_k8s_resources:
    #             if resource:
    #                 if as_yaml:
    #                     _yml = resource.get_k8s_manifest_yaml()
    #                     if _yml:
    #                         _resources.append(_yml)
    #                 else:
    #                     _json = resource.get_k8s_manifest_json(indent=2)
    #                     if _json:
    #                         _resources.append(_json)
    #
    #     if _resources:
    #         if as_yaml:
    #             print("---")
    #             print("---\n".join(_resources))
    #         else:
    #             print("\n".join(_resources))

    ######################################################
    ## Helpers
    ######################################################

    # def _debug_log(self, msg: Optional[str] = None) -> None:
    #     if msg:
    #         logger.debug(msg)
    #     kubeconfig_avl = True if self._kubeconfig else False
    #     k8s_client_avl = True if self.k8s_api_client else False
    #     logger.debug(f"Kubeconfig Available  : {kubeconfig_avl}")
    #     logger.debug(f"K8s_api Available  : {k8s_client_avl}")
    #
    # def verify_client(self) -> None:
    #     """Helper method to verify that we are good to perform K8s operaitons.
    #
    #     Raises:
    #         K8sClientException if something is wrong
    #     """
    #     if self._kubeconfig and self._k8s_client:
    #         pass
    #     else:
    #         self._debug_log()
    #         raise exceptions.K8sWorkerException("K8sWorker unavailable")


from django.shortcuts import render, get_object_or_404, redirect
from django.contrib.auth.mixins import LoginRequiredMixin, UserPassesTestMixin
from django.contrib.auth.models import User
from django.template import context
from django.views.generic import (
    ListView,
    DetailView,
    CreateView,
    UpdateView,
    DeleteView,
)
from django.http import Http404
from .models import Post
from django.contrib.auth.models  import User
from django.contrib import messages
from django.core.mail import send_mail
from django.conf import settings
from typing import Union
import logging
from django.contrib.auth.decorators import login_required

logger = logging.getLogger(__name__)

def home(request):
    #  lets create a dictionary and call it context, lets create a key named posts and the value of that key will be the data above in the posts list
    context = {
        'posts': Post.objects.all()
    }

    return render (request,'blog/home.html',context)

@login_required
def send_email(request, pk):
    user_email = User.objects.filter(username=request.user).first().email
    blog_post = Post.objects.filter(id=pk).first()

    if not user_email:
        logger.warning(f'Failed to send post {pk} via email due to missing user email')
        messages.warning(request, 'Make sure you are logged in as correct user with correct email address.')
    else:
        send_mail(
            subject=blog_post.title,
            message=blog_post.content,
            from_email=settings.EMAIL_HOST_USER,
            recipient_list=[user_email]
        )
        logger.info(f'Successfully sent post {pk} via email')
        messages.success(request, f'Successfully sent post to {user_email}')

    return redirect('post-detail', pk)

class PostListView(ListView):
    model = Post
    template_name = 'blog/home.html'   # <app>/<model>_<viewtype>.html
    context_object_name = 'posts'
    # date_posted order the list from oldest to newest, to reverse we add a - sign in the front
    ordering = ['-date_posted']
    paginate_by = 5

class UserPostListView(ListView):
    model = Post
    template_name = 'blog/user_posts.html'  # <app>/<model>_<viewtype>.html
    context_object_name = 'posts'
    paginate_by = 5

    def get_queryset(self):
        '''
            return query set consists of posts posted by user and ordered
            by posted date.

            Raise Not found error when user doesn't exist in User model
        '''
        try:
            user = get_object_or_404(User, username=self.kwargs.get('username'))
        except Http404:
            logger.error(f"Failed to find username {self.kwargs.get('username')} in User model.")
            messages.warning(self.request, "Something went wrong, can not view posts filtered by user.")
            raise Http404("User doesn't exist")
        return Post.objects.filter(author=user).order_by('-date_posted')

class PostDetailView(DetailView):
    model = Post
    fields = ["title", "content"]
    # When we are not defining template_name = <FILENAME>.html in our django defined class function, this means
    # that django will search for a template with default naming convention of # <app>/<model>_<viewtype>.html
    # In our PostDetailView function, it is looking for blog/post_detail.html
    # We had passed a template in PostListView since we had already created a template with another name and we
    # wanted django to search for it ,not the file with filename constructed using default naming convention.

class PostCreateView(LoginRequiredMixin, CreateView):
    # When we are not defining template_name = <FILENAME>.html in our django defined class function, this means
    # that django will search for a template with default naming convention of # <app>/<model>_form.html
    # In our PostCreateView & PostUpdateView function, it is looking for blog/post_form.html
    model = Post
    fields = ['title','content']

    def form_valid(self, form):
        form.instance.author = self.request.user
        return super().form_valid(form)

class PostUpdateView(LoginRequiredMixin, UserPassesTestMixin, UpdateView):
    model = Post
    fields = ['title','content']

    def form_valid(self, form):
        form.instance.author = self.request.user
        return super().form_valid(form)

    def test_func(self):
        return check_user_and_author_same(self)

class PostDeleteView(LoginRequiredMixin, UserPassesTestMixin, DeleteView):
    model = Post
    success_url = '/'

    def test_func(self):
        return check_user_and_author_same(self)

### Utility methods ###
def check_user_and_author_same(post_obj: Union[PostDeleteView, PostUpdateView]):
    '''
        Check user in request is same as author of post
    '''
    post = post_obj.get_object()
    if post_obj.request.user == post.author:
        return True
    return False


from django.shortcuts import render, HttpResponse
from app01 import models

# Create your views here.

def home(request):

    return HttpResponse("没错，我就是首页！")

def delete(request, title, id):
    # 删除语句
    # models.Book.objects.get(id=id).delete()

    # 数据库中保存的类名都是首字母大写的
    title = title.capitalize()

    # 从另一个文件根据字符串反射到具体的变量
    if hasattr(models, title):
        title_class = getattr(models, title)
        try:
            title_class.objects.get(id=id).delete()
        except Exception as e:
            print(str(e))
            print("id值不存在")
            return HttpResponse("名称：{} id:{}".format(title, id))
    else:
        return HttpResponse("表{}不存在".format(title))


from threading import Condition
from typing import List, Optional, Set

from ..backend import EventBackend

class StubBackend(EventBackend):
    def __init__(self) -> None:
        self.condition = Condition()
        self.events: Set[bytes] = set()

    def wait_many(self, keys: List[bytes], timeout: int) -> Optional[bytes]:
        with self.condition:
            if self.condition.wait_for(
                lambda: self._anyset(keys), timeout=timeout / 1000
            ):
                for key in keys:
                    if key in self.events:
                        self.events.remove(key)
                        return key
        return None

    def poll(self, key: bytes) -> bool:
        with self.condition:
            if key in self.events:
                self.events.remove(key)
                return True
        return False

    def notify(self, key: bytes, ttl: int) -> None:
        with self.condition:
            self.events.add(key)
            self.condition.notify_all()

    def _anyset(self, keys: List[bytes]) -> bool:
        return any(k in self.events for k in keys)


# -*- coding: utf-8 -*-
"""
Created on Sat Feb 24 18:33:39 2018

@author: patel
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as pl
from sklearn import preprocessing,cross_validation,svm
from sklearn.cross_validation import train_test_split
from sklearn.metrics import confusion_matrix
from sklearn.externals.six import StringIO
from sklearn import tree
from matplotlib import style
from matplotlib.colors import ListedColormap
style.use("ggplot")

dataset = pd.read_csv('test.csv')
dataset = dataset.replace(['male','female'],
[0,1])

#print(dataset)

# Seperating value into two objects
#X = dataset.iloc[:, [2,3,4]].values
#A = dataset.iloc[:, 17].values
X = dataset.iloc[:, :-1].values
y = dataset.iloc[:, 21].values
#dataset[X[:, 17]] = dataset[X[:, 17]].replace(0, dataset[X[:, 17]].mean())
#dataset[A] = dataset[A].replace(0, dataset[A].mean)

print("This is new age column replace with male and female into 0 and 1.")
#print(A)
print(X)
print(y)

print("This is the new value of gender column with missing values.")
print(X[:, 18])

# Replacing nan value with median in gender column which is 18th column

from sklearn.preprocessing import Imputer
imputer = Imputer(missing_values='NaN', strategy = 'median', axis = 0)
#X[:, [18,19]] = X[:,[18,19]].reshape(1, -1)
imputer = imputer.fit(X[:, [18,19]])
X[:, [18,19]] = imputer.transform(X[:, [18,19]])
print("This is gender column after replacing missing values with median function")
print(X[:, 18])
print("\n")

# Split the data into training and testing part
from sklearn.cross_validation import train_test_split
# Giving 20% of total dataset in testing part and remaining in training part (80%)

x_train, x_test, y_train, y_test = train_test_split(X,y,test_size=0.20, random_state = 0)

# Implementing Naive-Bayes Classifier algorithm for our dataset
from sklearn.naive_bayes import GaussianNB

#Create a Gaussian Classifier
model = GaussianNB()

# Train the model using the training sets
model.fit(x_train,y_train)

#Predict Output
predicted= model.predict(x_test)
print("This is the predicted data for testing X which contains 2000 datasets")
print(predicted)
print("\n")

accuracy = model.score(x_test, y_test)
print("This is the accuracy of Naive Bayes Classifier algoritham")
print(accuracy)
print("\n")

# Calculating confusion matrix for machine learning prediction and
cm= confusion_matrix(y_test, predicted)
print("The confusion matrix is described below.")
print(cm)


"""Unit tests for the openai LLM class"""

import pytest

from pandasai.llm import GoogleVertexAI
from pandasai.exceptions import UnsupportedModelError

class MockedCompletion:
    def __init__(self, result: str):
        self.result = result

class TestGoogleVertexAI:
    def test_init_with_default_model(self):
        project_id = "your_project_id"
        location = "northamerica-northeast1"
        vertexai_instance = GoogleVertexAI(project_id, location)

        assert vertexai_instance.model == "text-bison@001"
        assert vertexai_instance.project_id == project_id
        assert vertexai_instance.location == location

    def test_init_with_custom_model(self):
        project_id = "test-project"
        location = "northamerica-northeast1"
        custom_model = "code-bison@001"

        vertexai_instance = GoogleVertexAI(project_id, location, model=custom_model)

        assert vertexai_instance.model == custom_model
        assert vertexai_instance.project_id == project_id
        assert vertexai_instance.location == location

    @pytest.fixture
    def google_vertexai(self):
        # Create an instance of YourClass for testing
        project_id = "test-project"
        location = "northamerica-northeast1"
        custom_model = "code-bison@001"
        return GoogleVertexAI(project_id, location, custom_model)

    def test_validate_with_model(self, google_vertexai: GoogleVertexAI):
        google_vertexai.model = "text-bison@001"
        google_vertexai._validate()  # Should not raise any errors

    def test_validate_with_invalid_model(self, google_vertexai: GoogleVertexAI):
        google_vertexai.model = "invalid-model"
        with pytest.raises(
            UnsupportedModelError,
            match=(
                "Unsupported model: The model 'invalid-model' doesn't exist "
                "or is not supported yet."
            ),
        ):
            google_vertexai._generate_text("Test prompt")

    def test_validate_without_model(self, google_vertexai: GoogleVertexAI):
        google_vertexai.model = None
        with pytest.raises(ValueError, match="model is required."):
            google_vertexai._validate()


import logging
from pathlib import Path
from typing import Tuple, Union

import numpy as np
from matlatzinca.core.bn import BayesianNetwork
from matlatzinca.io import get_table_text
from matlatzinca.ui.dialogs import NotificationDialog
from PyQt5 import QtWidgets

logger = logging.getLogger(__name__)

class Project:
    def __init__(self, mainwindow) -> None:
        self.mainwindow = mainwindow
        self.bn = BayesianNetwork()

        self.signals = self.mainwindow.signals

        self.signals.node_coordinate_selected.connect(self.add_node)
        self.signals.node_about_to_be_removed.connect(self.remove_node)

        self.signals.edge_nodes_selected.connect(self.add_edge)
        self.signals.edge_about_to_be_removed.connect(self.remove_edge)
        self.signals.edge_about_to_be_reversed.connect(self.reverse_edge)

        # self.signals.delete_coordinate_selected.connect(self.add_node)

        self.signals.observed_correlation_about_to_change.connect(self.change_observed_correlation)
        self.signals.correlation_about_to_change.connect(self.change_edge_correlation)
        self.signals.name_about_to_change.connect(self.change_node_name)

        # self.signals.sampling_order_about_to_change.connect(self.change_sampling_order)
        self.signals.node_order_about_to_change.connect(self.change_node_order)
        self.signals.parent_order_about_to_change.connect(self.change_parent_order)

    def update_bn(self) -> None:
        """Updates the BN model by calculating the correlation matrix, correlation bounds, and edges"""
        if len(self.bn.nodes) == 0:
            self.bn.R = np.empty((0, 0), dtype=np.float64)
        else:
            logger.info("Calculating correlation matrix and bounds.")
            # self.bn.check_sampling_order()
            self.bn.calculate_correlation_matrix()
            self.bn.calculate_correlation_bounds()
            self.bn.create_edge_overview()

    def add_node(self, crd: Union[Tuple[float, float], None]) -> None:
        """Add node at specific coordinate"""
        if crd is None:
            return None

        self.signals.lists_about_to_change.emit()
        # Add node to BN
        name = self.bn._get_unused_name()
        self.bn.add_node(name=name, parents=[], rank_corrs=[])
        # Add coordinates to Node
        self.update_bn()

        logger.info(f'Node "{name}" added.')
        self.signals.lists_changed.emit()

        # Get the node and add coordinates
        node = self.bn._get_node_by_name(name)
        node.x, node.y = crd

        # Emit the signal that will update the plots
        self.signals.node_added.emit(crd, node)

    def remove_node(self, name: str) -> None:
        """Remove node by name"""
        self.signals.lists_about_to_change.emit()
        self.bn.remove_node(name)
        logger.info(f'Node "{name}" removed.')
        self.update_bn()
        self.signals.node_removed.emit(name)
        self.signals.lists_changed.emit()

    def remove_edge(self, parent: str, child: str) -> None:
        """Remove edge by parent and child name"""
        self.signals.lists_about_to_change.emit()
        self.bn.remove_edge(parent, child)
        logger.info(f'Edge from "{parent}" to "{child}" removed.')
        self.update_bn()
        self.signals.edge_removed.emit(parent, child)
        self.signals.lists_changed.emit()

    def add_edge(self, edge: Union[Tuple[str, str], None]) -> None:
        """Add edge from parent and child string"""
        if edge is None:
            return None

        self.signals.lists_about_to_change.emit()
        succes = self.bn.add_edge(*edge)
        if succes:
            logger.info(f'Edge from "{edge[0]}" to "{edge[1]}" added.')
            self.update_bn()
            # Emit the signal that will update the plots
            self.signals.edge_added.emit(*edge)

        else:
            NotificationDialog("Edge was not added, as the BN would no longer be a DAG.")
        self.signals.lists_changed.emit()

    def reverse_edge(self, parent: str, child: str) -> None:
        """Reverse an existing edge (and recalculate conditional rank correlations)"""

        self.signals.lists_about_to_change.emit()
        succes = self.bn.reverse_edge(parent, child)
        if succes:
            logger.info(f'Edge from "{parent}" to "{child}" reversed.')
            self.update_bn()
            # Emit the signal that will update the plots
            self.signals.edge_reversed.emit(parent, child)

        else:
            NotificationDialog("Edge was not reversed, as the BN would no longer be a DAG.")
        self.signals.lists_changed.emit()

    def change_observed_correlation(self, parent: str, child: str, corr: str) -> None:
        """Change (non-conditional) correlation coefficient"""

        node = self.bn._get_node_by_name(child)
        iparent = node.parent_index(parent)
        oldvalue = node.edges[iparent].rank_corr

        node.edges[iparent].rank_corr = corr
        corr = float(corr)

        # Check if the correlation is within bounds
        edge = node.edges[iparent]
        if corr < edge.rank_corr_bounds[0] or corr > edge.rank_corr_bounds[-1]:
            node.edges[iparent].rank_corr = oldvalue
            raise ValueError("Choose a value in between {:.4g} and {:.4g}".format(*edge.rank_corr_bounds))

        # Calculate the conditional correlation from the observed correlation
        cond_corr = self.bn.calculate_conditional_correlation(parent, child, observed=corr)

        # Change the conditional correlation
        self.change_edge_correlation(parent, child, corr=cond_corr)

    def change_edge_correlation(self, parent: str, child: str, corr: str) -> None:
        """Change the edge correlation"""

        node = self.bn._get_node_by_name(child)
        iparent = node.parent_index(parent)
        node.edges[iparent].cond_rank_corr = corr

        logger.info(f'Conditional edge ("{parent}" to "{child}") correlation changed to {corr}.')
        self.signals.lists_about_to_change.emit()
        self.update_bn()
        self.signals.correlation_changed.emit()
        self.signals.lists_changed.emit()

    def change_node_name(self, oldname: str, newname: str) -> None:
        """Change a node name"""

        if newname == "":
            return None
            # raise ValueError("Name cannot be empty")

        self.signals.lists_about_to_change.emit()

        self.bn.change_node_name(oldname, newname)

        logger.info(f'Node "{oldname}" was renamed to "{newname}".')

        self.signals.name_changed.emit()

        # self.bn.create_edge_overview()

        self.signals.lists_changed.emit()

    def change_node_order(self, source_pos: int, target_pos: int) -> None:
        """Change the node order"""

        self.signals.lists_about_to_change.emit()

        # Change the node order
        current_order = self.bn._node_names
        self.bn.nodes.insert(target_pos, self.bn.nodes.pop(source_pos))
        requested_order = self.bn._node_names

        # Reorder the nodes
        logger.info(f"Node order changed to to ({', '.join(requested_order)})")

        self.update_bn()

        self.signals.node_order_changed.emit()

        self.signals.lists_changed.emit()

    # def change_sampling_order(self, source_pos: int, target_pos: int) -> None:

    #     self.signals.lists_about_to_change.emit()

    #     # Change the node order
    #     self.bn.nodes.insert(target_pos, self.bn.nodes.pop(source_pos))
    #     requested_order = self.bn._node_names

    #     # Reorder the nodes based on the sampling order
    #     self.bn.check_sampling_order()
    #     valid_order = self.bn._node_names

    #     logger.info(f"Sampling order changed to to ({', '.join(valid_order)})")

    #     self.update_bn()

    #     self.signals.sampling_order_changed.emit()

    #     self.signals.lists_changed.emit()

    #     if requested_order != valid_order:
    #         NotificationDialog(
    #             f"The given order ({', '.join(requested_order)}) is not valid sampling order. Therefore, the order is changed to ({', '.join(valid_order)})"
    #         )

    def change_parent_order(self, source_pos: int, target_pos: int) -> None:
        """Change order of parents, as a result of reordering the edges"""
        self.signals.lists_about_to_change.emit()

        # Get edge from source position
        edge = self.bn.edgelist[source_pos]
        # Update order
        self.bn.change_parent_order(source_pos, target_pos)
        # Update BN with correlations
        self.update_bn()

        self.signals.parent_order_changed.emit()

        self.signals.lists_changed.emit()
        # Select the new row with the edge position
        self.signals.selected.emit((edge.parent, edge.child))

    def new(self) -> None:
        """Empty the project"""
        for node in reversed(self.bn.nodes):
            self.signals.node_about_to_be_removed.emit(node.name)

    def open(self, *args, fname: Path = None) -> None:
        """
        Method that loads project file and builds gui
        """

        if fname is None:
            fname = self.get_project_file()
            if fname == "":
                return None
            else:
                fname = Path(fname)

        # Clear current project
        self.new()

        # Open project
        self.mainwindow.setCursorWait()

        # Load from file
        self.signals.lists_about_to_change.emit()
        loaded_bn = BayesianNetwork.parse_file(fname)
        self.bn.nodes.extend(loaded_bn.nodes)
        self.update_bn()

        # Add nodes to UI
        for node in self.bn.nodes:
            self.signals.node_added.emit((np.random.rand(1)[0], np.random.rand(1)[0]), node)

        # Add edged to UI
        for node in self.bn.nodes:
            for edge in node.edges:
                self.signals.edge_added.emit(edge.parent, edge.child)

        self.signals.lists_changed.emit()

        # save current dir
        self.mainwindow.appsettings.setValue("currentdir", str(fname.parent))
        self.mainwindow.update_projectname(fname.name)
        self.signals.set_window_modified.emit(False)
        self.mainwindow.setCursorNormal()

    def save_as(self, fname=None) -> None:
        """
        Asks the user for the file name to save the project to.
        If the file name is passed, the user is not asked the file
        and the project is saved directly.

        Parameters
        ----------
        fname : str, optional
            File name, by default None
        """

        # if fname is None:
        # Set open file dialog settings
        options = QtWidgets.QFileDialog.Options()
        options |= QtWidgets.QFileDialog.DontUseNativeDialog
        options |= QtWidgets.QFileDialog.DontConfirmOverwrite

        # Set current dir
        currentdir = self.mainwindow.appsettings.value("currentdir", ".", type=str)

        # Open dialog to select file
        fname, ext = QtWidgets.QFileDialog.getSaveFileName(
            self.mainwindow, "Matlatzinca - Save project", currentdir, "JSON (*.json)", options=options
        )

        if fname == "":
            return None

        # Add extension if not given in file
        for end in [".json"]:
            if end in ext and not fname.endswith(end):
                fname += end

        # Convert to path
        fname = Path(fname)

        if fname.exists():
            reply = QtWidgets.QMessageBox.warning(
                self.mainwindow,
                "Matlatzinca - Save project",
                f"{fname.name} already exists.\nDo you want to overwrite it?",
                QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No,
            )
            if reply == QtWidgets.QMessageBox.No:
                return None

        # Save files
        self.mainwindow.update_projectname(fname.name)

        self.bn.to_json(fname)
        self.signals.set_window_modified.emit(False)

    def save(self) -> None:
        """
        Save current project, checks if the file name is already known.
        If not, the save as function is called and the user is asked to
        pick a file.
        """

        # If no current project, save as...
        currentproject = self.mainwindow.appsettings.value("currentproject", "", type=str)
        if currentproject == "":
            self.save_project_as()

        else:
            # Get current directory
            currentdir = Path(self.mainwindow.appsettings.value("currentdir", ".", type=str))

            # Else, save files directly
            self.bn.to_json(currentdir / currentproject)
            self.signals.set_window_modified.emit(False)

    def get_project_file(self) -> str:
        """
        Opens a dialog to select a project file to open.
        """
        # Set open file dialog settings
        options = QtWidgets.QFileDialog.Options()
        options |= QtWidgets.QFileDialog.DontUseNativeDialog

        # Set current dir
        currentdir = self.mainwindow.appsettings.value("currentdir", ".", type=str)

        # Open dialog to select file
        fname, _ = QtWidgets.QFileDialog.getOpenFileName(
            self.mainwindow,
            "Matlatzinca - Open project",
            currentdir,
            "JSON (*.json)",
            options=options,
        )

        return fname

    def export(self, what: str, how: str) -> None:
        """Exports correlation matrix or tables to csv or clipboard

        Args:
            what (str): correlation_matrix, nodes, or edges
            how (str): csv or clipboard
        """

        if not what in ["correlation_matrix", "nodes", "edges"]:
            raise KeyError(
                f'Choose one of "{", ".join(["correlation_matrix", "nodes", "edges"])}" for "what"'
            )

        if not how in ["csv", "clipboard"]:
            raise KeyError(f'Choose one of "{", ".join(["csv" , "clipboard"])}" for "how"')

        sep = "\t" if how == "clipboard" else ","

        # Gather data
        if what == "correlation_matrix":
            data = [self.bn._node_names] + self.bn.R[:, :].astype(str).tolist()
            data = [[name] + row for row, name in zip(data, [""] + self.bn._node_names)]
            text = "\n".join([sep.join(row) for row in data])

        elif what == "nodes":
            text = get_table_text(model=self.mainwindow.nodes_edges_widget.nodemodel, newline_replace=" ")

        elif what == "edges":
            text = get_table_text(model=self.mainwindow.nodes_edges_widget.edgemodel, newline_replace=" ")

        # Export data
        if how == "csv":
            # Set open file dialog settings
            options = QtWidgets.QFileDialog.Options() | QtWidgets.QFileDialog.DontUseNativeDialog

            # Set current dir
            currentdir = self.mainwindow.appsettings.value("currentdir", ".", type=str)

            # Open dialog to select file
            fname, ext = QtWidgets.QFileDialog.getSaveFileName(
                self.mainwindow, "BN Drawing - Save data", currentdir, "CSV (*.csv)", options=options
            )

            if fname == "":
                return None
            elif ".csv" in ext and not fname.endswith(".csv"):
                fname += ".csv"

            # Convert to path
            fname = Path(fname)
            with fname.open("w", encoding="utf-8", errors=" ") as f:
                f.write(text)

        elif how == "clipboard":
            QtWidgets.qApp.clipboard().setText(text)


from django.conf.urls import patterns, include, url
from rest_framework.urlpatterns import format_suffix_patterns
from django.contrib import admin
from listme.views import api, lists, usersettings

admin.autodiscover()

urlpatterns = patterns('',
                       url(r'^api/$', api.api_root),
                       url(
                           r'^api/lists/$', api.ListList.as_view(),
                           name='list-list'
                       ),
                       url(r'^api/lists/(?P<pk>\d+)/$', api.ListDetail.as_view()),
                       url(
                           r'^api/users/$', api.UserList.as_view(),
                           name='user-list'
                       ),
                       url(r'^api/users/(?P<pk>\d+)/$', api.UserDetail.as_view()),
                       url(r'^lists/$', lists.lists_handler),
                       url(r'^lists/(?P<pk>\d+)/$', lists.lists_handler),
                       url(r'^u/(?P<user_pk>\d+)/lists$', lists.lists_handler),
                       url(r'^u/(?P<user_pk>\d+)/lists/(?P<list_pk>\d+)$', lists.lists_handler),
                       url(r'^u/(?P<user_pk>\d+)/settings$', usersettings.settings_handler),
                       url(r'^admin/', include(admin.site.urls)),

)

urlpatterns = format_suffix_patterns(urlpatterns)

urlpatterns += [
    url(r'^api-login/', include('rest_framework.urls',
                               namespace='rest_framework')),
]


import plotly.graph_objs as go
import requests
import datetime

def get_exchange_rate(date, currency_name):
    response = requests.get(
        f'https://api.privatbank.ua/p24api/exchange_rates?json&date={date}'
    )  # Делаем запрос на API Приватбанка и получаем Response - ответ, кладем его в переменную response
    dct = response.json()
    currencies = dct['exchangeRate']

    for currency_dict in currencies:
        if 'currency' in currency_dict:
            if currency_dict['currency'] == currency_name:
                return currency_dict["saleRateNB"]

    return False

currency_name = 'RUB'

dates = []
exchange_rates = []

year = 2014
while year <= 2020:
    print(f'Оброблюється {year} рік...')
    date = f"01.01.{year}"
    rate = get_exchange_rate(date, currency_name)
    dates.append(date)
    exchange_rates.append(rate)
    year += 1

print('Оброблюється сьогоднішнє число')
date_ = datetime.datetime.now()
date = f"{date_.day}.{date_.month}.{date_.year}"
rate = get_exchange_rate(date, currency_name)
dates.append(date)
exchange_rates.append(rate)

diag = go.Bar(x=dates, y=exchange_rates, name=f'Ціни валюти {currency_name} (у грн)')
go.Figure(data=[diag]).write_html('exchange_ratesRUB.html', auto_open=True)


import datetime
import cloud
import constants
import build

"""
Function returns the time taken (in hrs) to repair a failed build
"""
def getMeanTimeToRepair():
    data = []

    for i in constants.TEAMCITY_BUILD_DETAILS():
        data += getMeanTime(i["ID"], i["Name"])

    cloud.uploadToCloud('FixTime.csv', data)

    return data

def getMeanTime(buildID, name):
    data = build.getTeamCityData(buildID, name)

    # Reverse the list to arrange data in ascending chronological order
    data = data[::-1]

    res = []

    for i in range(len(data)):
        if data[i]["finalStatus"] == "Failure" and notAnotherFailure(data, i):
            indexLimit = len(data) - 1
            # Edge Case where latest build is a failure
            if i >= indexLimit:
                break
            else:
                index = i
                numberOfBuilds = 0

                while index < indexLimit and data[index]["finalStatus"] != "Success":
                    index += 1
                    numberOfBuilds += 1

                res.append(generateData(data, i, index, numberOfBuilds))

    return res

"""
Function returns true if the previous build and the current build are both a failure.
This prevents double counting the number of failures if there are consecutive build failures
"""
def notAnotherFailure(data, index):
    if index == 0:
        return True
    else:
        if data[index - 1]["finalStatus"] == "Failure":
            return False
        else:
            return True

def generateData(data, failIndex, successIndex, numberOfBuilds):
    res = {
        "Deployment": data[failIndex]["name"],
        "End Time of Failed Build": data[failIndex]["finishDate"],
        "End Time of Successful Build": data[successIndex]["finishDate"],
        "Number of Builds Taken to Fix": numberOfBuilds,
        "Sprint": data[failIndex]["sprint"],
        "Build ID": data[failIndex]["id"],
        "Link to Build": f'https://teamcity.gahmen.tech/viewLog.html?buildId={data[failIndex]["id"]}&buildTypeId=bgp_appian_deploy_03bDeployGccQa',
        "Hours Taken": findTimeTaken(data[failIndex]["finishDate"], data[successIndex]["finishDate"])
    }

    return res

def findTimeTaken(startTime, endTime):
    startTime = startTime.replace('T', '').replace('+', '')
    startTime = datetime.datetime.strptime(startTime, '%Y%m%d%H%M%S%f')

    endTime = endTime.replace('T', '').replace('+', '')
    endTime = datetime.datetime.strptime(endTime, '%Y%m%d%H%M%S%f')

    diff = endTime - startTime

    seconds = diff.seconds

    hours = seconds / 3600

    return round(hours, 2)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Apr 27 20:57:37 2021

@author: akimlavrinenko
"""

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.font_manager import FontProperties

#params
n = 10 #number of the bins
step = 5
x0 = -0.5
y0 = -0.5
z0 = -0.5

box_coords = [[0 for x in range(n+1)] for x in range(3)]
box_node = [[0 for x in range(n)] for x in range(3)]

box_coords[0][0] = x0
box_coords[1][0] = y0
box_coords[2][0] = z0

delta = 1/n
path = '/home/akim/coding/data/sphere_trajectories'

for j in range(0,3):
    for i in range(1,n+1):
        box_coords[j][i] = box_coords[j][i-1] + delta
        box_node[j][i-1] = (box_coords[j][i-1] + box_coords[j][i])/2

box_coords = np.asarray(np.transpose(box_coords))
box_node = np.asarray(np.transpose(box_node))
box_node = np.round(box_node, 3)

legend = box_node.tolist()

cm = plt.get_cmap('tab20')

for n in range(10):
    data = np.genfromtxt(path + '/sphere_trajectory_' + str(n) + '.dat')
    print(n, np.shape(data))
    x = data[::step,1::3]
    y = data[::step,2::3]
    z = data[::step,3::3]
    time = data[::step, 0]
    d2List = []
    # for t in range(len(time)):
    for t in range(10):
        print(t)
        ll = []
        for num in range(len(x[0,:])):
            xd = [(number - x[t,num])**2 for number in x[t,:]]
            yd = [(number - y[t,num])**2 for number in y[t,:]]
            zd = [(number - z[t,num])**2 for number in z[t,:]]
            ld = [xd,yd,zd]
            sd = [sum(i) for i in zip(*ld)]
            ll.append(sd)
        s = [sum(i) for i in zip(*ll)]
        d2 = sum(s)/len(x[0,:])
        d2List.append(d2)
        # print(d2)

        plt.plot(x[t,:],y[t,:], '.')
        plt.xlim(-0.5,0.5)
        plt.ylim(-0.5,0.5)
        plt.show()
    # plt.plot(np.log10(time), np.log10(d2List))
    # plt.xlabel('log t, [s]')
    # plt.ylabel('log D^2, [m^2]')
    # plt.grid(True)
    # plt.tight_layout()
    # plt.legend(legend, title='location', bbox_to_anchor=(0.9, 0.67))
# plt.savefig('log10_relative_dispersion.png', dpi=150)