import os

from HdRezkaApi import *
import requests

def download_video(directory, file_name, link):
    r = requests.get(link, stream=True)

    directory_exists = os.path.exists(directory)

    if not directory_exists:
        os.makedirs(directory)

    with open("{}/{}.mp4".format(directory, file_name), 'wb') as f:
        for chunk in r.iter_content(chunk_size=1024 * 1024):
            if chunk:
                f.write(chunk)

url = input('Enter url: ')

rezka = HdRezkaApi(url)

seasons = rezka.getSeasons()
translations = list(rezka.getTranslations().keys())

qualities = list(rezka.getStream(1, 1).videos.keys())

for i in range(0, len(qualities)):
    print("{}. {}".format(i + 1, qualities[i]))

user_choice = input("\nSelect quality: ")
quality = qualities[int(user_choice) - 1]

print("\n\n")

for i in range(0, len(translations)):
    print("{}. {}".format(i + 1, translations[i]))

user_choice = input("\nSelect translation: ")
translation = translations[int(user_choice) - 1]

seasons_count = len(seasons[translation]['seasons'])

film_name = url.split('/')[-1].split('.')[0]

for i in range(1, seasons_count + 1):
    print("Downloading season {}/{}".format(i, seasons_count))

    episode_count = 0

    while True:
        try:
            episode_count += 1
            print("Downloading episode {}".format(episode_count))

            name = "{}-{}".format(i, episode_count)

            download_video(film_name, name, rezka.getStream(i, episode_count, translation)(quality))
        except:
            break


from argparse import ArgumentParser

def get_divisors(n):
    divisors = []
    for i in range(1, int(n/2) + 1):
        if n % i == 0:
            divisors.append(i)
    return divisors

def are_amicable(n, m):
    n_divisors = get_divisors(n)
    if sum(n_divisors) == m:
        m_divisors = get_divisors(m)
        if sum(m_divisors) == n:
            return True
    return False

def main():
    parser = ArgumentParser()
    parser.add_argument('n', help="First number")
    parser.add_argument('m', help="Second number")
    args = parser.parse_args()
    n = int(args.n)
    m = int(args.m)

    if are_amicable(n, m):
        print(f"{n} and {m} are so-called amicable numbers - "
              f"which means that the sum of the proper divisors of one number is the other number, and vice versa.")
    else:
        print(f"Sorry, {n} and {m} are not amicable numbers :(")

if __name__ == '__main__':
    main()


import requests
from bs4 import BeautifulSoup
import pandas as pd
import markdownify
import re
from tqdm import tqdm
from datasets import Dataset
import concurrent.futures
import os

import itertools

class LinksParser:
    def __init__(
        self,
    ) -> None:
        self.domain = "https://www.povarenok.ru/recipes"
        self.dataset = []

    def parse_func(self, page_num):
        url = f"{self.domain}/~{page_num}/?sort=rating&order=desc"
        response = requests.get(url=url)
        soup = BeautifulSoup(response.text)
        items = soup.find_all("article", class_="item-bl")
        dataset = []

        for card in items:
            title = card.find("h2").text.strip()
            ingridients = [
                item.text for item in card.find("span", class_="list").find_all("a")
            ]
            views = int(card.find("span", class_="i-views").text)
            likes = int(card.find("span", class_="i-likes").text)
            ups = int(card.find("span", class_="i-up").text)
            link = card.find("h2").find("a")["href"]

            dataset.append(
                {
                    "title": title,
                    "ingridients": ingridients,
                    "views": views,
                    "likes": likes,
                    "ups": ups,
                    "link": link,
                }
            )
        return dataset

    def parse_parallel(self):
        with concurrent.futures.ProcessPoolExecutor(
            max_workers=os.cpu_count()
        ) as executor:
            texts = tqdm(
                executor.map(
                    self.parse_func,
                    list(range(1, 3101)),
                    chunksize=1,
                )
            )
            texts = list(texts)
        self.dataset = list(itertools.chain(*texts))

if __name__ == "__main__":
    parser = LinksParser()
    parser.parse_parallel()
    # print(parser.dataset)
    dataset = Dataset.from_list(parser.dataset)
    dataset.push_to_hub("dim/povarenok_links")


# -*- coding: utf-8 -*-
########################################################################################################
# AI人工智障写作 - https://github.com/BlinkDL/AI-Writer
########################################################################################################

import json

import numpy as np
import paddle

import paddle_src.utils
from paddle_src.model import GPT, GPTConfig

# paddle_src.utils.set_seed(42) # 是否固定随机数（固定后每次运行的生成结果都一样）

print("\nPaddle2.x版本AI人工智障写作 https://github.com/BlinkDL/AI-Writer")
print("注：这是使用paddle框架改写后的AI人工智障写作！！！！！！！！！！！！！")
print("请关注原作者的知乎 https://zhuanlan.zhihu.com/p/394766831")
print("\n声明：模型的训练数据全部来自网文，缺乏生活常识。生成的文字仅供娱乐。请遵守法律法规。")

# 需 paddle 2.2.x 及以上版本

MODEL_NAME = "model/wangwen-2022-02-15"  # 模型名
WORD_NAME = "model/wangwen-2022-02-15"  # 这个也修改

NUM_OF_RUNS = 999  # 写多少遍
LENGTH_OF_EACH = 512  # 每次写多少字

top_p = 0.75  # 这个的范围是 0 到 1。越大，变化越多。越小，生成效果越规矩。自己试试 0 和 0.5 和 1.0 的效果就知道了
top_p_newline = 0.9

# 开头非常重要。开头需创造剧情点。开头文笔越好，续写就越好。开头乱写，续写也乱写。
# 开头这样输入：
# context = "魔法"
# context = "“区区"
# context = "三体舰队"
context = "这是一颗"
# context = "众人一惊，没想到这林黛玉的剑法竟如此精妙，只见在那剑影下，剑尖朝着伏地魔的脖子探去，眼见避无可避，伏地魔情急，大喊"

# 多行的开头这样输入：
# context = """
# 这几天心里颇不宁静。今晚在院子里坐着乘凉，忽然想起日日走过的荷塘，在这满月的光里，总该另有一番样子吧。月亮渐渐地升高了，墙外马路上孩子们的欢笑，已经听不见了；妻在屋里拍着闰儿，迷迷糊糊地哼着眠歌。我悄悄地披了大衫，带上门出去。
# 沿着荷塘，是一条曲折的小煤屑路。这是一条幽僻的路；白天也少人走，夜晚更加寂寞。荷塘四面，长着许多树，蓊蓊郁郁的。路的一旁，是些杨柳，和一些不知道名字的树。没有月光的晚上，这路上阴森森的，有些怕人。今晚却很好，虽然月光也还是淡淡的。
# 路上只我一个人，背着手踱着。这一
# """

##############################################################################

ctx_len = 512
n_layer = 12
n_head = 12
n_embd = n_head * 64
n_attn = n_embd
n_ffn = n_embd

context = context.strip().split("\n")
for c in range(len(context)):
    context[c] = context[c].strip().strip("\u3000")
context = "\n" + ("\n".join(context)).strip()
print("您输入的开头有 " + str(len(context)) + " 个字。注意，模型只会看最后 " + str(ctx_len) + " 个字。")

with open(WORD_NAME + ".json", "r", encoding="utf-16") as result_file:
    word_table = json.load(result_file)

vocab_size = len(word_table)

train_dataset = lambda: None
train_dataset.stoi = {v: int(k) for k, v in word_table.items()}
train_dataset.itos = {int(k): v for k, v in word_table.items()}
UNKNOWN_CHAR = train_dataset.stoi["\ue083"]

print("\nLoading model...")
model = GPT(
    GPTConfig(
        vocab_size,
        ctx_len,
        n_layer=n_layer,
        n_head=n_head,
        n_embd=n_embd,
        n_attn=n_attn,
        n_ffn=n_ffn,
    )
)
model.set_state_dict(paddle.load(MODEL_NAME + ".pdparams"))

model.eval()

print("done:", MODEL_NAME, "&", WORD_NAME)

##############################################################################

for run in range(NUM_OF_RUNS):

    x = np.array(
        [train_dataset.stoi.get(s, UNKNOWN_CHAR) for s in context], dtype=np.int64
    )

    real_len = len(x)
    print_begin = 0

    for i in range(LENGTH_OF_EACH):

        if i == 0:

            print(("-" * 60) + "\n" + context.replace("\n", "\n  ").strip("\n"), end="")
            print_begin = real_len

        with paddle.no_grad():
            xxx = paddle.to_tensor(x[-ctx_len:], dtype="int64").unsqueeze(0)
            out = model(xxx)[0]
            out[:, :, UNKNOWN_CHAR] = -10000.0
        pos = -1 if real_len >= ctx_len else real_len - 1

        if train_dataset.itos[int(x[real_len - 1])] == "\n":
            char = paddle_src.utils.sample_logits(
                out, pos, temperature=1.0, top_p=top_p_newline
            )
        else:
            char = paddle_src.utils.sample_logits(
                out, pos, temperature=1.0, top_p=top_p
            )

        x = np.append(x, char)
        real_len += 1

        if i % 2 == 1 or i == LENGTH_OF_EACH - 1 or i < 10:
            completion = "".join(
                [train_dataset.itos[int(i)] for i in x[print_begin:real_len]]
            )
            print(completion.replace("\n", "\n  "), end="", flush=True)
            print_begin = real_len
    print()


import json
try:
    from json import JSONDecodeError
except ImportError:
    JSONDecodeError = ValueError

from kubernetes.client.models import V1Container

from scan.fetchers.cli.cli_fetcher import CliFetcher
from scan.fetchers.kube.kube_access import KubeAccess
from base.utils.exceptions import CredentialsError, HostAddressError, SshError

class KubeFetchContainers(KubeAccess, CliFetcher):

    PROXY_ATTR = 'kube-proxy'
    SANDBOX_ID_ATTR = 'io.kubernetes.sandbox.id'
    ATTRIBUTES_TO_IGNORE = ['resources', 'liveness_probe', 'readiness_probe', 'security_context']

    def get(self, parent_id) -> list:
        pod_id = parent_id.replace('-containers', '')
        pod_obj = self.inv.get_by_id(self.get_env(), pod_id)
        if not pod_obj:
            self.log.error('inventory has no pod with uid={}'.format(pod_id))
            return []

        # TODO: temporary
        labels = pod_obj.get('labels', {})
        if labels and labels.get('calipso-rancher-pod-for-kube-proxy'):
            return [self.get_rancher_proxy_container(pod_obj)]

        host = pod_obj['host']
        pod_filter = 'spec.nodeName={}'.format(host)
        pods = self.api.list_pod_for_all_namespaces(field_selector=pod_filter)
        ret = []
        if not pods or len(pods.items) == 0:
            self.log.error('failed to find pod with nodeName={}'.format(host))
            return []

        pod = next((pod for pod in pods.items if pod.metadata.uid == pod_id),
                   None)
        if not pod:
            self.log.error('failed to find pod with uid={}'.format(pod_id))
            return []

        for container in pod.spec.containers:
            ret.append(self.get_container(container, pod, pod_obj))
        return ret

    def get_rancher_proxy_container(self, pod_obj):
        doc = {
            'type': 'container',
            'namespace': pod_obj['namespace'],
            'host': pod_obj['host'],
            'pod': {'id': pod_obj['id'], 'name': pod_obj['object_name']},
            'id': '{}-kube-proxy'.format(pod_obj['id']),
            'name': 'kube-proxy',
            'container_id': 'kube-proxy'
        }
        self.update_container_config(doc=doc, pod_obj=pod_obj)
        self.update_interface_link(doc=doc, pod_obj=pod_obj)
        self.update_proxy_container_info(container=doc, pod_obj=pod_obj)
        return doc

    def get_container(self, container, pod, pod_obj):
        doc = {
            'type': 'container',
            'environment': self.get_env(),
            'namespace': pod.metadata.namespace,
            **self.get_container_data(container)
        }
        self.update_container_status_data(doc, pod_obj)
        doc.update({
            'host': pod_obj['host'],
            'pod': {
                'id': pod_obj['id'],
                'name': pod_obj['object_name']
            },
            'ip_address': pod_obj.get('pod_status', {}).get('pod_ip', ''),
            'id': '{}-{}'.format(pod_obj['id'], doc['name'])
        })
        if doc.get('image'):
            doc['image'] = {"name": doc['image']}
        self.update_container_config(doc=doc, pod_obj=pod_obj)
        self.update_interface_link(doc=doc, pod_obj=pod_obj)
        self.update_proxy_container_info(container=doc, pod_obj=pod_obj)
        return doc

    @staticmethod
    def _get_state_from_container_status(container_status):
        for status_key in ('waiting', 'running', 'terminated'):
            if container_status.get('state', {}).get(status_key):
                return status_key

    def update_container_status_data(self, doc: dict, pod_obj: dict):
        container_statuses = pod_obj['pod_status']['container_statuses']
        container_status = next(s for s in container_statuses
                                if s['name'] == doc['name'])
        if not container_status:
            self.log.error('failed to find container_status record '
                           'for container {} in pod {}'
                           .format(doc['name'], pod_obj['name']))
            return
        container_id = container_status['container_id']
        if container_id is None:
            doc['container_type'] = container_status['name']
            doc['container_id'] = container_status['image']
        else:
            id_parts = container_id.split('://')
            doc['container_type'] = id_parts[0]
            doc['container_id'] = id_parts[1]
        doc['container_status'] = container_status
        doc['state'] = self._get_state_from_container_status(container_status)

    @classmethod
    def get_container_data(cls, container: V1Container):
        return cls.class_to_dict(data_object=container, exclude=cls.ATTRIBUTES_TO_IGNORE)

    def update_container_config(self, doc, pod_obj):
        if doc.get('state') == 'waiting':
            return

        cmd = 'docker inspect {}'.format(doc['container_id'])
        try:
            output = self.run(cmd, ssh_to_host=pod_obj['host'])
            data = json.loads(output)
        except SshError as e:
            self.log.warning('"docker inspect" cmd failed for container {}. '
                             'Error: {}'.format(doc['container_id'], e))
            return
        except JSONDecodeError as e:
            self.log.error('error reading output of cmd: {}, {}'
                           .format(cmd, str(e)))
            return
        if not data:
            return

        data = data[0]
        if 'State' in data:
            doc['container_state'] = data['State']
            doc['state'] = data['State']['Status']  # Prefer actual state to the one fetched from container_status
        if 'Config' in data:
            doc['config'] = data['Config']
            self.update_container_sandbox(doc=doc, pod_obj=pod_obj)

    def update_container_sandbox(self, doc: dict, pod_obj: dict) -> None:
        sandbox_id = doc['config'].get('Labels').get(self.SANDBOX_ID_ATTR)
        if not sandbox_id:
            return

        cmd = 'docker inspect {}'.format(sandbox_id)
        output = self.run(cmd, ssh_to_host=pod_obj['host'])
        try:
            data = json.loads(output)
        except JSONDecodeError as e:
            self.log.error('error reading output of cmd: {}, {}'
                           .format(cmd, str(e)))
            return
        if not data:
            return

        doc['sandbox'] = data[0]
        self.find_network(doc)

    def update_interface_link(self, doc: dict, pod_obj: dict):
        if doc.get('state') != 'running' or doc['namespace'] == 'cattle-system' or doc['name'] == 'kubernetes-dashboard':
            doc['vnic_index'] = ''
            return

        interface_name = 'vpp1' if 'VPP' in self.configuration.environment['mechanism_drivers'] else 'eth0'
        file_name = "/sys/class/net/{}/iflink".format(interface_name)
        cmd = 'docker exec {0} sh -c "test -f {1} && cat {1} || exit 0"'.format(doc['container_id'], file_name)
        try:
            output = self.run(cmd, ssh_to_host=pod_obj['host'])
            doc['vnic_index'] = output.strip()
            self.add_container_to_vnic(container=doc, pod_obj=pod_obj)
        except (SshError, CredentialsError, HostAddressError):
            doc['vnic_index'] = ''

    # find network matching the one sandbox, and keep its name
    def find_network(self, doc):
        networks = doc['sandbox']['NetworkSettings']['Networks']
        if not networks:
            return
        if isinstance(networks, dict):
            network_names = list(networks.keys())
            network = network_names[0]
            network = networks[network]
        else:
            network = networks[0]
        network_id = network['NetworkID']
        network_obj = self.inv.get_by_id(self.get_env(), network_id)
        if not network_obj:
            return
        doc['network'] = network_id

    def add_container_to_vnic(self, container: dict, pod_obj: dict) -> None:
        condition = {
            'environment': self.get_env(),
            'type': 'vnic',
            'host': pod_obj['host'],
        }
        if 'VPP' in self.configuration.environment['mechanism_drivers']:
            condition['ip_address'] = container['ip_address']
        else:
            condition['index'] = container['vnic_index']
        vnic = self.inv.find_one(condition)
        if not vnic:
            return

        # TODO: figure out this logic
        # re-calc new ID and name path for vNIC and vNICs folder
        # self.set_folder_parent(vnic, object_type='vnic',
        #                        master_parent_id=container['id'],
        #                        master_parent_type='container')
        vnic_containers = vnic.get('containers', [])
        vnic_container = {'id': container['container_id'], 'name': container['name']}
        vnic['containers'] = vnic_containers.append(vnic_container) if vnic_containers else [vnic_container]
        self.inv.set(vnic)

        # self.inv.save_inventory_object(vnic,
        #                                parent=container,
        #                                environment=self.get_env())

    def update_proxy_container_info(self, container, pod_obj):
        if container['name'] != self.PROXY_ATTR:
            return
        container['container_app'] = self.PROXY_ATTR
        self.update_proxy_container_config(container)
        self.update_proxy_nat_tables(container)
        container['vservices'] = self.get_proxy_container_vservices(pod_obj)
        self.add_proxy_container_to_vservices(container)

    def update_proxy_container_config(self, container: dict) -> None:
        command = container.get('command')
        if not command or not isinstance(command, list) or len(command) < 2:
            self.log.error('unable to find {} command file '
                           'for container {}'
                           .format(self.PROXY_ATTR, container['id']))
            return
        conf_line = command[1]
        if not isinstance(conf_line, str) \
                or not conf_line.startswith('--config='):
            self.log.error('unable to find {} command config file '
                           'for container {}'
                           .format(self.PROXY_ATTR, container['id']))
            return
        conf_file = conf_line[len('--config='):]
        cmd = 'docker exec {} cat  {}'.format(container['container_id'],
                                              conf_file)
        conf_file_contents = self.run(cmd=cmd, ssh_to_host=container['host'])
        container['kube_proxy_config'] = conf_file_contents

    def update_proxy_nat_tables(self, container: dict) -> None:
        cmd = 'docker exec {} iptables -t nat -n -L' \
            .format(container['container_id'])
        nat_tables = self.run(cmd=cmd, ssh_to_host=container['host'])
        container['nat_tables'] = nat_tables

    def get_proxy_container_vservices(self, pod_obj: dict) -> list:
        pods = self.inv.find_items({
            'environment': self.get_env(),
            'type': 'pod',
            'host': pod_obj['host'],
            'vservices': {'$exists': 1}
        })
        vservices = []
        for pod in pods:
            vservices.extend(pod['vservices'])
        return vservices

    def add_proxy_container_to_vservices(self, container: dict) -> None:
        for vservice in list(container.get('vservices', [])):
            self.add_proxy_container_to_vservice(container=container, vservice=vservice)

    def add_proxy_container_to_vservice(self, container: dict, vservice: dict) -> None:
        vservice_obj = self.inv.get_by_id(self.get_env(), vservice['id'])
        if not vservice_obj:
            self.log.error('failed to find vservice object with id {} '
                           'in container {} (id: {})'
                           .format(vservice['id'],
                                   container['object_name'],
                                   container['id']))
        if self.PROXY_ATTR not in vservice_obj:
            vservice_obj[self.PROXY_ATTR] = []
        matches = [p for p in vservice_obj[self.PROXY_ATTR]
                   if p['id'] == container['id']]
        if not matches:
            proxy_data = dict(id=container['id'], name=container['name'],
                              host=container['host'])
            vservice_obj[self.PROXY_ATTR].append(proxy_data)
            self.inv.set(vservice_obj)


#!/usr/bin/env python2
#
# download_aemo.py: download AEMO dispatch, price and demand data.
#
# Copyright (c) 2014 Cameron Patrick <cameron@largestprime.net>
#
# This file is part of AusEnergyViz. AusEnergyViz is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, see <http://www.gnu.org/licenses/>.

import urllib3
import re
import os
import sys
import datetime
import argparse
import socket

# base URLs for AEMO/NEM data
URL_DISPATCH_CURRENT = 'http://www.nemweb.com.au/REPORTS/CURRENT/Dispatch_SCADA/'
URL_DISPATCH_ARCHIVE = 'http://www.nemweb.com.au/REPORTS/ARCHIVE/Dispatch_SCADA/'
URL_DISPATCH_SWIS = 'http://data.imowa.com.au/datafiles/facility-scada/'
URL_PRICEDEMAND = 'http://www.nemweb.com.au/mms.GRAPHS/data/'
RETRY_LIMIT = 5

# List of regions (states) for price/demand data. Each state has a start date
# which is the first month for which data is available, and an optional finish
# date which is the last month for which data is available.
STATES_PRICEDEMAND = [
        ('NSW1', (1998, 12), None),
        ('QLD1', (1998, 12), None),
        ('SA1', (1998, 12), None),
        ('SNOWY1', (1998, 12), (2008, 6)),
        ('VIC1', (1998, 12), None),
        ('TAS1', (2005, 5), None)
    ]

def fetch_url(url):
    """Returns the contents of a URL as a string."""
    retry = 0
    while True:
        try:
            http = urllib3.PoolManager()
            response = http.urlopen('GET', url)
            content = response.data
            break
        except (urllib3.exceptions, socket.error) as e:
            retry = retry + 1
            if retry > RETRY_LIMIT:
                sys.stderr.write("***FAILED*** reason: %s\n" % str(e.reason))
                sys.exit(1)
    return content

def extract_regexp_set(page, regexp):
    """Returns an alphabetically sorted list of matches to a regular expression.
    Useful for finding file names in a blob of HTML, for example.
    """
    return sorted(set(re.findall(regexp, page)))

def archived_zip_exists(z, archive_list):
    """Given a 'long' dispatch zip name (for a 5 minute data point), see
    if there is a matching daily archive zip present in the given set.
    """
    date_match = re.search(r'(?i)_([0-9_]+).zip', z)
    if date_match is None:
        return False
    date_str = date_match.group(1)
    archive_file = 'PUBLIC_DISPATCHSCADA_%s.ZIP' % date_str[0:8]
    return archive_file in archive_list

def fetch_aemo_zips(index_url, dir, dir_archive=None):
    """Downloads AEMO dispatch zip files from the AEMO web site. Zip files which
    have already been downloaded are skipped. Zip files containing 5min data
    where we already have the daily archive file are not downloaded, and any
    existing ones are deleted.

    Args:
        index_url: the URL of the base directory on the AEMO web site to
            download zip files from
        dir: the path to save dispatch data to
        dir_archive: if present, the path to archived dispatch data so we know
            what 5min data can be skipped or deleted
    """
    sys.stderr.write("%s: " % index_url)
    index_html = fetch_url(index_url)

    zip_list = extract_regexp_set(index_html, r'(?i)PUBLIC_DISPATCHSCADA_[0-9_]+.zip')
    sys.stderr.write("found %d zips.\n" % len(zip_list))

    if not os.path.isdir(dir):
        os.makedirs(dir)

    # make a list of archived/aggregated zips so we can skip duplicates
    if dir_archive is not None:
        archive_list = set(s.upper() for s in os.listdir(dir_archive))
    else:
        archive_list = set()

    existing_list = set(z.upper() for z in os.listdir(dir))

    skipped = 0
    for z in zip_list:
        path = os.path.join(dir, z)

        # skip non-aggregate files if we have the aggregated version
        if archived_zip_exists(z, archive_list):
            skipped += 1
            continue

        # skip files that already exist (case insensitive)
        if z.upper() in existing_list:
            skipped += 1
            continue

        # download from web site
        sys.stderr.write("Downloading: %s " % z)
        zip_content = fetch_url(index_url + z)

        # save to file
        f = file(path, 'wb')
        f.write(zip_content)
        f.close()

        sys.stderr.write("[%d bytes]\n" % len(zip_content))
    sys.stderr.write("Skipped %d zips.\n" % skipped)

    # clean old zip files
    for filename in os.listdir(dir):
        if archived_zip_exists(filename, archive_list):
            sys.stderr.write("Deleting:    %s\n" % filename)
            os.unlink(os.path.join(dir, filename))

def fetch_aemo_pricedemand(url, path, state, first, last=None):
    """Download price and demand data from the AEMO web site.

    Args:
        url: the base URL of the price/demand CSVs to download
        state: string containing the NEM state or region to download
        first: tuple (year,month) of the first month to download
        last: tuple (year,month) of the last month to download, or None to
            use the current date.
    """
    if last is None:
        today = datetime.date.today()
        last = (today.year, today.month)

    if not os.path.isdir(path):
        os.makedirs(path)

    cur = None
    while True:
        # advance to next month
        if cur is None:
            cur = first
        else:
            # increment month
            cur = (cur[0], cur[1] + 1)
            # if month passes 12, increment year
            if cur[1] > 12:
                cur = (cur[0] + 1, 1)
        # have we reached the current month yet?
        if cur >= last:
            break

        # determine name of file to download, skip if it exists
        filename = 'DATA%.4d%.2d_%s.csv' % (cur[0], cur[1], state)
        outfile = os.path.join(path, filename)
        if os.path.exists(outfile):
            continue

        # download from web site
        sys.stderr.write("Downloading: %s ..." % filename)
        csv_content = fetch_url(url + filename)

        # save to file
        f = file(outfile, 'wb')
        f.write(csv_content)
        f.close()
        sys.stderr.write(" done\n")

def fetch_swis_dispatch(url, path):
    """Download dispatch data for Western Australia (SWIS)."""

    if not os.path.isdir(path):
        os.makedirs(path)
    existing_list = set(s.lower() for s in os.listdir(path))

    # the most recent CSV file we have might have changed since last download.
    # if any newer CSV files are present, we assume that older ones are up-to-date.
    last_existing = None
    if len(existing_list) > 0:
        last_existing = sorted(existing_list, reverse=True)[0]

    sys.stderr.write("%s: " % url)
    index_html = fetch_url(url)
    csv_list = extract_regexp_set(index_html, r'(?i)facility-scada-[0-9]+-[0-9]+.csv')
    sys.stderr.write("found %d CSVs.\n" % len(csv_list))

    skipped = 0
    for z in csv_list:
        outfile = os.path.join(path, z)

        # skip files that already exist (case insensitive)
        if z.lower() in existing_list and z.lower() != last_existing:
            skipped += 1
            continue

        # download from web site
        sys.stderr.write("Downloading: %s " % z)
        csv_content = fetch_url(url + z)

        # save to file
        f = file(outfile, 'wb')
        f.write(csv_content)
        f.close()

        sys.stderr.write("[%d bytes]\n" % len(csv_content))
    sys.stderr.write("Skipped %d CSVs.\n" % skipped)

if __name__ == '__main__':
    # parse arguments
    parser = argparse.ArgumentParser(description='Download AEMO dispatch and demand data.')
    parser.add_argument('path_base', metavar='PATH',
            help='base directory to store downloaded data in')
    args = parser.parse_args()

    # determine output directories based on our base path
    path_archive = os.path.join(args.path_base, 'dispatch_daily')
    path_current = os.path.join(args.path_base, 'dispatch_5min')
    path_pricedemand = os.path.join(args.path_base, 'pricedemand')
    path_swis_dispatch = os.path.join(args.path_base, 'dispatch_swis')

    # download dispatch data - first daily archives, then 5 minute current data
    fetch_aemo_zips(URL_DISPATCH_ARCHIVE, path_archive)
    fetch_aemo_zips(URL_DISPATCH_CURRENT, path_current, path_archive)

    # download price and demand data
    for state, first, last in STATES_PRICEDEMAND:
        fetch_aemo_pricedemand(URL_PRICEDEMAND, path_pricedemand, state, first, last)

    # download SWIS (IMOWA) dispatch data
    fetch_swis_dispatch(URL_DISPATCH_SWIS, path_swis_dispatch)


#!/usr/bin/python3
import boto3
import sys
from creds import Creds

creds = Creds("credentials.csv")
#print("Beware You are doing this activity using this account :" + " " + creds.username)

REGION = "ap-south-1"
ACCESS_KEY = creds.access_key_id
SECRET_KEY = creds.secret_key
ec2 = boto3.resource('ec2', aws_access_key_id=ACCESS_KEY, aws_secret_access_key=SECRET_KEY, region_name=REGION)

instance_ids = ec2.instances.all()
for id in instance_ids:
 print(id.terminate())


import sqlite3

MIN = 1
MAX = 5
CREATE = 1
READ = 2
UPDATE = 3
DELETE = 4
QUIT = 5

def main():
    choice = 0
    while choice != QUIT:
        display_menu()
        choice = get_menu_choice()
        if choice == CREATE:
            create()
        elif choice == READ:
            read()
        elif choice == UPDATE:
            update()
        elif choice == DELETE:
            delete()

def display_menu():
    print('--------Menu---------')
    print('1. Create.')
    print('2. Read.')
    print('3. Update.')
    print('4. Delete.')
    print('5. Quit.')

def get_menu_choice():
    choice = int(input('Choose your option: '))

    while choice < MIN or choice > MAX:
        print(f'Please choose option between {MIN} and {MAX}.')
        choice = int(input('Choose your option: '))

    return choice

def create():
    print('Create a new position.')
    name = input('The name of position: ')
    price = input('Price: ')
    insert_row(name, price)

def read():
    name = input('Search for position: ')
    num_found = display_item(name)
    print(f'{num_found} row(s) found.')

def update():
    read()
    selected_id = int(input('Choose position to update: '))
    name = input('Enter new name: ')
    price = input('Enter new price: ')
    num_updated = update_row(selected_id, name, price)
    print(f'{num_updated} row(s) updated.')

def delete():
    read()
    selected_id = int(input('Choose position to delete: '))
    sure = input('Are you sure you want to delete this position? y/n\n')
    if sure.lower() == 'y':
        num_deleted = delete_row(selected_id)
        print(f'{num_deleted} row(s) deleted.')

def insert_row(name, price):
    conn = None
    try:
        conn = sqlite3.connect('Data/inventory.db')
        cur = conn.cursor()
        cur.execute('''INSERT INTO Inventory (ItemName, Price)
                        VALUES (?,?)''',
                    (name, price))
        conn.commit()
    except sqlite3.Error as err:
        print('Database error', err)
    finally:
        if conn != None:
            conn.close()

def display_item(name):
    conn = None
    result = []
    try:
        conn = sqlite3.connect('Data/inventory.db')
        cur = conn.cursor()
        cur.execute('''SELECT * FROM Inventory
                    WHERE ItemName == ?''',
                    (name, ))
        result = cur.fetchall()
        for row in result:
            print(f'ID: {row[0]:<3} Name: {row[1]:<15}'
                  f'Price:  {row[2]:<6}')
    except sqlite3.Error as err:
        print('Database error', err)
    finally:
        if conn != None:
            conn.close()
        return len(result)

def update_row(id, name, price):
    conn = None
    try:
        conn = sqlite3.connect('Data/inventory.db')
        cur = conn.cursor()
        cur.execute('''UPDATE Inventory
                        SET ItemName == ?, Price ==?
                        WHERE ItemID == ?''',
                    (name, price, id))
        conn.commit()
        num_updated = cur.rowcount
    except sqlite3.Error as err:
        print('Database error', err)
    finally:
        if conn != None:
            conn.close()
        return num_updated

def delete_row(id):
    conn = None
    try:
        conn = sqlite3.connect('Data/inventory.db')
        cur = conn.cursor()
        cur.execute('''DELETE FROM Inventory
                        WHERE ItemID == ?''',
                    (id,))
        conn.commit()
        num_deleted = cur.rowcount
    except sqlite3.Error as err:
        print('Database error', err)
    finally:
        if conn != None:
            conn.close()
        return num_deleted

if __name__ == '__main__':
    main()


import glob
# import seaborn as sns
import logging
import logging.handlers
import os
import platform
import sys
import argparse
import traceback
import warnings
from builtins import enumerate
from sklearn.metrics import mean_squared_error
from flaml import AutoML
from lightgbm import plot_importance

# import datetime as dt
# from datetime import datetime
# import pvlib
import matplotlib.dates as mdates
import matplotlib.cm as cm
# from sklearn.metrics import mean_absolute_error

import matplotlib as mpl
import matplotlib.pyplot as  plt
import numpy as np
from scipy.stats import linregress
import pandas as pd
import math
from scipy import spatial
from pandas.tseries.offsets import Day, Hour, Minute, Second
from scipy.interpolate import Rbf
from numpy import zeros, newaxis

# os.environ["PROJ_LIB"] = "C:\ProgramData\Anaconda3\Library\share"
# from pygem import IDW
# import eccodes
import pygrib
# import pykrige.kriging_tools as kt
import pytz
import requests
import datetime
import h2o
# from pycaret.regression import *
from sqlalchemy import create_engine
import re
import configparser
import sqlalchemy
from sqlalchemy.ext.declarative import declarative_base
import sqlalchemy
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import create_engine, MetaData, Table, Column, Integer, String, DateTime, Float
from sqlalchemy.orm import sessionmaker
from sqlalchemy import create_engine
import pymysql
import re
import configparser
import sqlalchemy
from sqlalchemy.ext.declarative import declarative_base
from scipy.stats import linregress
import argparse
from sklearn.model_selection import train_test_split
import numpy as np
import pandas as pd
# from sklearn.model_selection import train_test_split

import xarray as xr
from pvlib import location
from pvlib import irradiance

import h2o
from h2o.automl import H2OAutoML

from matplotlib import font_manager, rc

try:
    from pycaret.regression import *
except Exception as e:
    print("Exception : {}".format(e))

try:
    from pycaret.regression import *
except Exception as e:
    print("Exception : {}".format(e))

import pandas as pd
import numpy as np
from sklearn.preprocessing import MinMaxScaler
from keras import regularizers
from sklearn.metrics import mean_absolute_percentage_error, mean_squared_error
from sklearn.ensemble import RandomForestRegressor
import xgboost
# from sklearn.utils import safe_indexing

import os
import pickle
import ray

# =================================================
# 사용자 매뉴얼
# =================================================
# [소스 코드의 실행 순서]
# 1. 초기 설정 : 폰트 설정
# 2. 유틸리티 함수 : 초기화 함수 (로그 설정, 초기 변수, 초기 전달인자 설정) 또는 자주 사용하는 함수
# 3. 주 프로그램 :부 프로그램을 호출
# 4. 부 프로그램 : 자료 처리를 위한 클래스로서 내부 함수 (초기 변수, 비즈니스 로직, 수행 프로그램 설정)
# 4.1. 환경 변수 설정 (로그 설정) : 로그 기록을 위한 설정 정보 읽기
# 4.2. 환경 변수 설정 (초기 변수) : 입력 경로 (inpPath) 및 출력 경로 (outPath) 등을 설정
# 4.3. 초기 변수 (Argument, Option) 설정 : 파이썬 실행 시 전달인자 설정 (pyhton3 *.py argv1 argv2 argv3 ...)
# 4.4. 비즈니스 로직 수행 : 단위 시스템 (unit 파일명)으로 관리 또는 비즈니스 로직 구현

# =================================================
# 1. 초기 설정
# =================================================
warnings.filterwarnings("ignore")
# font_manager._rebuild()

plt.rc('font', family='Malgun Gothic')
plt.rc('axes', unicode_minus=False)
# sns.set(font="Malgun Gothic", rc={"axes.unicode_minus": False}, style='darkgrid')

# 그래프에서 마이너스 글꼴 깨지는 문제에 대한 대처
mpl.rcParams['axes.unicode_minus'] = False

# 타임존 설정
tzKst = pytz.timezone('Asia/Seoul')
tzUtc = pytz.timezone('UTC')
dtKst = datetime.timedelta(hours=9)

# =================================================
# 2. 유틸리티 함수
# =================================================
# 로그 설정
def initLog(env=None, contextPath=None, prjName=None):
    if env is None: env = 'local'
    if contextPath is None: contextPath = os.getcwd()
    if prjName is None: prjName = 'test'

    saveLogFile = "{}/{}_{}_{}_{}.log".format(
        contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'log', prjName)
        , platform.system()
        , platform.node()
        , prjName
        , datetime.datetime.now().strftime("%Y%m%d")
    )

    if not os.path.exists(os.path.dirname(saveLogFile)):
        os.makedirs(os.path.dirname(saveLogFile))

    # logger instance 생성
    log = logging.getLogger(prjName)

    if len(log.handlers) > 0:
        return log

    # format 생성
    format = logging.Formatter('%(asctime)s [%(name)s | %(lineno)d | %(filename)s] [%(levelname)-5.5s] %(message)s')

    # handler 생성
    streamHandler = logging.StreamHandler()
    fileHandler = logging.FileHandler(saveLogFile)

    # logger instance에 format 설정
    streamHandler.setFormatter(format)
    fileHandler.setFormatter(format)

    # logger instance에 handler 설정
    log.addHandler(streamHandler)
    log.addHandler(fileHandler)

    # logger instance로 log 기록
    log.setLevel(level=logging.INFO)

    return log

#  초기 변수 설정
def initGlobalVar(env=None, contextPath=None, prjName=None):
    if env is None: env = 'local'
    if contextPath is None: contextPath = os.getcwd()
    if prjName is None: prjName = 'test'

    # 환경 변수 (local, 그 외)에 따라 전역 변수 (입력 자료, 출력 자료 등)를 동적으로 설정
    # 즉 local의 경우 현재 작업 경로 (contextPath)를 기준으로 설정
    # 그 외의 경우 contextPath/resources/input/prjName와 같은 동적으로 구성
    globalVar = {
        'prjName': prjName
        , 'sysOs': platform.system()
        , 'contextPath': contextPath
        , 'resPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources')
        , 'cfgPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'config')
        , 'inpPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'input', prjName)
        , 'figPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'fig', prjName)
        , 'outPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'input', prjName)
        , 'movPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'movie', prjName)
        , 'logPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'log', prjName)
        , 'mapPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'mapInfo')
        , 'sysPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'config', 'system.cfg')
        , 'seleniumPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'config', 'selenium')
        , 'fontPath': contextPath if env in 'local' else os.path.join(contextPath, 'resources', 'config', 'fontInfo')
    }

    return globalVar

#  초기 전달인자 설정
def initArgument(globalVar, inParams):
    # 원도우 또는 맥 환경
    if globalVar['sysOs'] in 'Windows' or globalVar['sysOs'] in 'Darwin':
        inParInfo = inParams

        # 글꼴 설정
        plt.rc('font', family='Malgun Gothic')

    # 리눅스 환경
    if globalVar['sysOs'] in 'Linux':
        parser = argparse.ArgumentParser()

        for i, argv in enumerate(sys.argv[1:]):
            if not argv.__contains__('--'): continue
            parser.add_argument(argv)

        inParInfo = vars(parser.parse_args())

        # 글꼴 설정
        fontList = glob.glob('{}/{}'.format(globalVar['fontPath'], '*.ttf'))
        fontName = font_manager.FontProperties(fname=fontList[0]).get_name()
        plt.rcParams['font.family'] = fontName

    log.info("[CHECK] inParInfo : {}".format(inParInfo))

    for key, val in inParInfo.items():
        if val is None: continue
        # 전역 변수에 할당
        globalVar[key] = val

    # 전역 변수
    for key, val in globalVar.items():
        if env not in 'local' and key.__contains__('Path') and env and not os.path.exists(val):
            os.makedirs(val)

        globalVar[key] = val.replace('\\', '/')

        log.info("[CHECK] {}".format(val))

        # self 변수에 할당
        # setattr(self, key, val)

    return globalVar

# 딥러닝 예측
def makeDlModel(subOpt=None, xCol=None, yCol=None, inpData=None):

    log.info('[START] {}'.format('makeDlModel'))
    log.info('[CHECK] subOpt : {}'.format(subOpt))

    result = None

    try:

        saveModel = '{}/{}/{}-{}-{}-{}-{}-{}.model'.format(globalVar['modelPath'], subOpt['modelKey'], serviceName, subOpt['srvId'], 'final', 'h2o', 'for', '*')
        saveModelList = sorted(glob.glob(saveModel), reverse=True)
        xyCol = xCol.copy()
        xyCol.append(yCol)
        data = inpData[xyCol]
        dataL1 = data.dropna()

        # h2o.shutdown(prompt=False)

        if (not subOpt['isInit']):
            h2o.init()
            h2o.no_progress()
            subOpt['isInit'] = True

        # 학습 모델이 없을 경우
        if (subOpt['isOverWrite']) or (len(saveModelList) < 1):

            # 7:3에 대한 학습/테스트 분류
            # trainData, validData = train_test_split(dataL1, test_size=0.3)

            # dlModel = H2OAutoML(max_models=30, max_runtime_secs=99999, balance_classes=True, seed=123)
            dlModel = H2OAutoML(max_models=20, max_runtime_secs=99999, balance_classes=True, seed=123)

            #dlModel.train(x=xCol, y=yCol, training_frame=h2o.H2OFrame(trainData), validation_frame=h2o.H2OFrame(validData))
            #dlModel.train(x=xCol, y=yCol, training_frame=h2o.H2OFrame(dataL1), validation_frame=h2o.H2OFrame(dataL1))
            dlModel.train(x=xCol, y=yCol, training_frame=h2o.H2OFrame(dataL1), validation_frame=h2o.H2OFrame(dataL1))

            fnlModel = dlModel.get_best_model()
            #fnlModel = dlModel.leader

            # 학습 모델 저장
            saveModel = '{}/{}/{}-{}-{}-{}-{}-{}.model'.format(globalVar['modelPath'], subOpt['modelKey'], serviceName, subOpt['srvId'], 'final', 'h2o', 'for', datetime.datetime.now().strftime('%Y%m%d'))
            log.info('[CHECK] saveModel : {}'.format(saveModel))
            os.makedirs(os.path.dirname(saveModel), exist_ok=True)

            # h2o.save_model(model=fnlModel, path=os.path.dirname(saveModel), filename=os.path.basename(saveModel), force=True)
            fnlModel.save_mojo(path=os.path.dirname(saveModel), filename=os.path.basename(saveModel), force=True)
        else:
            saveModel = saveModelList[0]
            log.info('[CHECK] saveModel : {}'.format(saveModel))
            fnlModel = h2o.import_mojo(saveModel)

        result = {
            'msg': 'succ'
            , 'dlModel': fnlModel
            , 'saveModel': saveModel
            , 'isExist': os.path.exists(saveModel)
        }

        return result

    except Exception as e:
        log.error('Exception : {}'.format(e))
        return result

    finally:
        # try, catch 구문이 종료되기 전에 무조건 실행
        log.info('[END] {}'.format('makeDlModel'))

# 머신러닝 예측
def makeMlModel(subOpt=None, xCol=None, yCol=None, inpData=None):

    log.info('[START] {}'.format('makeMlModel'))
    log.info('[CHECK] subOpt : {}'.format(subOpt))

    result = None

    try:
        saveModel = '{}/{}/{}-{}-{}-{}-{}-{}.model.pkl'.format(globalVar['outPath'], subOpt['modelKey'], serviceName, subOpt['srvId'], 'final', 'pycaret', 'for', '*')
        saveModelList = sorted(glob.glob(saveModel), reverse=True)
        xyCol = xCol.copy()
        xyCol.append(yCol)
        data = inpData[xyCol]
        dataL1 = data.dropna()

        # 학습 모델이 없을 경우
        if (subOpt['isOverWrite']) or (len(saveModelList) < 1):

            # 7:3에 대한 학습/테스트 분류
            # trainData, validData = train_test_split(data, test_size=0.3)
            # trainData = inpData

            # from pycaret.regression import *
            # from pycaret.regression import setup
            # from pycaret.regression import compare_models

            # from sklearn.impute import SimpleImputer

            dataL1['is_weekday'] = dataL1['is_weekday'].astype('category')
            dataL1['is_spring'] = dataL1['is_spring'].astype('category')
            dataL1['is_summer'] = dataL1['is_summer'].astype('category')
            dataL1['is_fall'] = dataL1['is_fall'].astype('category')
            dataL1['is_winter'] = dataL1['is_winter'].astype('category')

            # dataL2 = dataL1[['CA_TOT', 'SG_PWRER_USE_AM']]

            pyModel = setup(
                data=dataL1
                , session_id=123
                , silent=True
                , target=yCol
                # 2022.11.02
                # , normalize=True
            )

            # pyModel = setup(
            #     data=dataL1
            #     , session_id=123
            #     , silent=True
            #     , target=yCol
            #     , remove_outliers= True
            #     , remove_multicollinearity = True
            #     , ignore_low_variance = True
            #     , normalize=True
            #     , transformation= True
            #     , transform_target = True
            #     , combine_rare_levels = True
            # )

            # 각 모형에 따른 자동 머신러닝
            # modelList = compare_models(sort='RMSE', n_select=3)
            # modelList = compare_models(sort='RMSE', n_select=2)
            modelList = compare_models(sort='RMSE')

            # 앙상블 모형
            # blendModel = blend_models(estimator_list=modelList, fold=10)

            # 앙상블 파라미터 튜닝
            # tuneModel = tune_model(blendModel, fold=10, choose_better=True)

            # 학습 모형
            fnlModel = finalize_model(modelList)

            # 학습 모형 저장
            saveModel = '{}/{}/{}-{}-{}-{}-{}-{}.model'.format(globalVar['outPath'], subOpt['modelKey'], serviceName, subOpt['srvId'], 'final', 'pycaret', 'for', datetime.datetime.now().strftime('%Y%m%d'))
            log.info('[CHECK] saveModel : {}'.format(saveModel))
            os.makedirs(os.path.dirname(saveModel), exist_ok=True)
            save_model(fnlModel, saveModel)

        else:
            saveModel = saveModelList[0]
            log.info('[CHECK] saveModel : {}'.format(saveModel))
            fnlModel = load_model(os.path.splitext(saveModel)[0])

        result = {
            'msg': 'succ'
            , 'mlModel': fnlModel
            , 'saveModel': saveModel
            , 'isExist': os.path.exists(saveModel)
        }

        return result

    except Exception as e:
        log.error('Exception : {}'.format(e))
        return result

    finally:
        # try, catch 구문이 종료되기 전에 무조건 실행
        log.info('[END] {}'.format('makeMlModel'))

### 주말이면 1, 평일이면 0 반환
def check_weekend(date):
    days = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
    day = date.weekday()
    if days[day] in ['Sat', 'Sun']:
        return 1
    else:
        return 0

### 봄 여부
def check_spring(date):
    month = date.month
    if month in [3, 4, 5]:
        return 1
    else:
        return 0

### 여름 여부
def check_summer(date):
    month = date.month
    if month in [6, 7, 8]:
        return 1
    else:
        return 0

### 가을 여부
def check_fall(date):
    month = date.month
    if month in [9, 10, 11]:
        return 1
    else:
        return 0

### 겨울 여부
def check_winter(date):
    month = date.month
    if month in [12, 1, 2]:
        return 1
    else:
        return 0

# 머신러닝 예측
def makeFlamlModel(subOpt=None, xCol=None, yCol=None, trainData=None, testData=None):
    log.info('[START] {}'.format('makeFlamlModel'))
    log.info('[CHECK] subOpt : {}'.format(subOpt))

    result = None

    try:
        saveModel = '{}/{}/{}-{}-{}-{}-{}-{}.model'.format(globalVar['outPath'], subOpt['modelKey'], serviceName, subOpt['srvId'], 'final', 'flaml', 'for', '*')
        saveModelList = sorted(glob.glob(saveModel), reverse=True)
        xyCol = xCol.copy()
        xyCol.append(yCol)

        trainDataL1 = trainData[xyCol]
        trainDataL2 = trainDataL1.dropna()

        testDataL1 = testData[xyCol]
        testDataL2 = testDataL1.dropna()

        # 학습 모델이 없을 경우
        if (subOpt['isOverWrite']) or (len(saveModelList) < 1):

            # 7:3에 대한 학습/테스트 분류
            # trainData, validData = train_test_split(dataL1, test_size=0.3)

            # 전체 학습 데이터
            # trainData = dataL1

            # flModel = MultiOutputRegressor(
            #     AutoML(
            #         task="regression"
            #         , metric='rmse'
            #         , time_budget=60
            #     )
            # )

            ray.init(num_cpus=12, ignore_reinit_error=True)

            flModel = AutoML(
                task="regression"
                , metric='rmse'
                # , time_budget=600
                # , early_stop = True
                , ensemble = False
                # , ensemble = True
                , seed = 123
                , time_budget=60
            )

            # 각 모형에 따른 자동 머신러닝
            flModel.fit(X_train=trainData[xCol], y_train=trainData[yCol], n_jobs=12)
            # flModel.fit(X_train=trainData[xCol], y_train=trainData[yCol], n_jobs=4, n_concurrent_trials=4)
            # flModel.fit(X_train=trainData[xCol], y_train=trainData[yCol], n_jobs=4)
            # flModel.fit(X_train=trainDataL2[xCol], y_train=trainDataL2[yCol], X_test=testDataL2[xCol], y_test=testDataL2[yCol])
            # flModel.fit(X_train=trainDataL2[xCol], y_train=trainDataL2[yCol])

            # 학습 모형 저장
            saveModel = '{}/{}/{}-{}-{}-{}-{}-{}.model'.format(globalVar['outPath'], subOpt['modelKey'], serviceName, subOpt['srvId'], 'final', 'flaml', 'for', datetime.datetime.now().strftime('%Y%m%d'))
            log.info('[CHECK] saveModel : {}'.format(saveModel))
            os.makedirs(os.path.dirname(saveModel), exist_ok=True)

            with open(saveModel, 'wb') as f:
                pickle.dump(flModel, f, pickle.HIGHEST_PROTOCOL)

        else:
            saveModel = saveModelList[0]
            log.info('[CHECK] saveModel : {}'.format(saveModel))

            with open(saveModel, 'rb') as f:
                flModel = pickle.load(f)

        result = {
            'msg': 'succ'
            , 'mlModel': flModel
            , 'saveModel': saveModel
            , 'isExist': os.path.exists(saveModel)
        }

        return result

    except Exception as e:
        log.error('Exception : {}'.format(e))
        return result

    finally:
        # try, catch 구문이 종료되기 전에 무조건 실행
        log.info('[END] {}'.format('makeFlamlModel'))

def short_term_predict(raw_data, CNSMR_NO):
    print("Predict {} Start!!".format(CNSMR_NO))
    data = raw_data.loc[raw_data['CNSMR_NO'] == CNSMR_NO, ['SRV', 'MESURE_DATE_TM', 'CA_TOT', 'HM', 'PA', 'TA', 'TD', 'WD', 'WS', 'SZA', 'AZA', 'ET', 'TURB', 'GHI_CLR', 'DNI_CLR', 'DHI_CLR', 'SWR', 'SG_PWRER_USE_AM']].reset_index(drop=True)
    data['MESURE_DATE_TM'] = pd.to_datetime(data['MESURE_DATE_TM'])
    data['is_weekday'] = data['MESURE_DATE_TM'].map(check_weekend)
    data['is_spring'] = data['MESURE_DATE_TM'].map(check_spring)
    data['is_summer'] = data['MESURE_DATE_TM'].map(check_summer)
    data['is_fall'] = data['MESURE_DATE_TM'].map(check_fall)
    data['is_winter'] = data['MESURE_DATE_TM'].map(check_winter)
    data = data.loc[:, ['CA_TOT', 'HM', 'PA', 'TA', 'TD', 'WD', 'WS', 'SZA', 'AZA', 'ET', 'TURB', 'GHI_CLR', 'DNI_CLR', 'DHI_CLR', 'SWR', 'SG_PWRER_USE_AM', 'is_weekday', 'is_spring', 'is_summer', 'is_fall', 'is_winter']]
    # 독립변수 목록 정의
    independent_variable = ['CA_TOT', 'HM', 'PA', 'TA', 'TD', 'WD', 'WS', 'SZA', 'AZA', 'ET', 'TURB', 'GHI_CLR', 'DNI_CLR', 'DHI_CLR', 'SWR', 'is_weekday', 'is_spring', 'is_summer', 'is_fall', 'is_winter']
    minmax_scaler = MinMaxScaler()
    minmax_scaler = minmax_scaler.fit_transform(data.loc[:, independent_variable])
    data.loc[:, independent_variable] = minmax_scaler

    ### 단기 예측
    t = 48
    X = np.array(data.loc[:, independent_variable])
    y = data['SG_PWRER_USE_AM']

    train_X = X[:-t, :]
    test_X = X[-t:, :]
    train_y = np.array(y[:-t])
    test_y = np.array(y[-t:])

    filename = './model/short_term/' + CNSMR_NO + '.model'
    model = pickle.load(open(filename, 'rb'))

    test_predict = model.predict(test_X)
    # print("RMSE: {}".format(np.sqrt(mean_squared_error(test_y, test_predict))))
    # print("MAPE: {}".format(mean_absolute_percentage_error(test_y, test_predict)))
    return train_X, test_X, train_y, test_y, model

# ================================================
# 4. 부 프로그램
# ================================================
class DtaProcess(object):
    # ================================================
    # 요구사항
    # ================================================
    # Python을 이용한 한국 월별 수출액과 코스피 종간 간의 회귀분석

    # ================================================================================================
    # 환경변수 설정
    # ================================================================================================
    global env, contextPath, prjName, serviceName, log, globalVar

    # env = 'local'  # 로컬 : 원도우 환경, 작업환경 (현재 소스 코드 환경 시 .) 설정
    env = 'dev'  # 개발 : 원도우 환경, 작업환경 (사용자 환경 시 contextPath) 설정
    # env = 'oper'  # 운영 : 리눅스 환경, 작업환경 (사용자 환경 시 contextPath) 설정

    if (platform.system() == 'Windows'):
        contextPath = os.getcwd() if env in 'local' else 'E:/04. TalentPlatform/Github/TalentPlatform-Python'
    else:
        contextPath = os.getcwd() if env in 'local' else '/SYSTEMS/PROG/PYTHON/PyCharm'

    prjName = 'test'
    serviceName = 'PRJ2022'

    # 4.1. 환경 변수 설정 (로그 설정)
    log = initLog(env, contextPath, prjName)

    # 4.2. 환경 변수 설정 (초기 변수)
    globalVar = initGlobalVar(env, contextPath, prjName)

    # ================================================================================================
    # 4.3. 초기 변수 (Argument, Option) 설정
    # ================================================================================================
    def __init__(self, inParams):

        log.info("[START] __init__ : {}".format("init"))

        try:
            # 초기 전달인자 설정 (파이썬 실행 시)
            # pyhton3 *.py argv1 argv2 argv3 ...
            initArgument(globalVar, inParams)

        except Exception as e:
            log.error("Exception : {}".format(e))
            raise e
        finally:
            log.info("[END] __init__ : {}".format("init"))

    # ================================================================================================
    # 4.4. 비즈니스 로직 수행
    # ================================================================================================
    def exec(self):

        log.info('[START] {}'.format("exec"))

        try:

            if (platform.system() == 'Windows'):

                # 옵션 설정
                sysOpt = {
                    # 시작/종료 시간
                    # 'srtDate': '2021-10-01'
                    # , 'endDate': '2021-10-10'
                    'srtDate': '2019-01-01'
                    , 'endDate': '2022-05-22'
                    # , 'endDate': '2021-11-01'

                    , 'stnId': '1'

                    #  딥러닝
                    , 'dlModel': {
                        # 초기화
                        'isInit': False

                        # 모형 업데이트 여부
                        , 'isOverWrite': True
                        #, 'isOverWrite': False

                        # 모형 버전 (날짜)
                        , 'modelVer': '*'
                    }

                    #  머신러닝
                    , 'mlModel': {
                        # 모델 업데이트 여부
                        # 'isOverWrite': True
                        'isOverWrite': False

                        # 모형 버전 (날짜)
                        , 'modelVer': '*'
                    }
                }

            else:

                # 옵션 설정
                sysOpt = {
                    # 시작/종료 시간
                    # 'srtDate': globalVar['srtDate']
                    # , 'endDate': globalVar['endDate']
                    # , 'stnId': globalVar['stnId']
                    'srtDate': '2019-01-01'
                    , 'endDate': '2022-05-22'
                    , 'stnId': '1'

                    #  딥러닝
                    , 'dlModel': {
                        # 초기화
                        'isInit': False

                        # 모형 업데이트 여부
                        , 'isOverWrite': True
                        #, 'isOverWrite': False

                        # 모형 버전 (날짜)
                        , 'modelVer': '*'
                    }

                    #  머신러닝
                    , 'mlModel': {
                        # 모델 업데이트 여부
                        'isOverWrite': True
                        #'isOverWrite': False

                        # 모형 버전 (날짜)
                        , 'modelVer': '*'
                    }
                }

            globalVar['inpPath'] = '/DATA/INPUT'
            globalVar['outPath'] = '/DATA/OUTPUTT'
            globalVar['figPath'] = '/DATA/FIG'

            # ******************************************************************************
            # 크몽 전문가 검증 테스트
            # ******************************************************************************
            inpFile = '{}/{}/{}.csv'.format(globalVar['inpPath'], serviceName, 'FINAL_PW_DATA')
            fileList = sorted(glob.glob(inpFile))
            # pw_data = pd.read_csv('FINAL_PW_DATA.csv')
            pw_data = pd.read_csv(fileList[0])
            pw_data['MESURE_DATE_TM'] = pd.to_datetime(pw_data['MESURE_DATE_TM'])
            # pw_data.columns

            # inpFile = '{}/{}/{}.csv'.format(globalVar['inpPath'], serviceName, 'FINAL_FORE_DATA')
            inpFile = '{}/{}/{}.csv'.format(globalVar['inpPath'], serviceName, 'FINAL_FORE_DATA_20221110')
            fileList = sorted(glob.glob(inpFile))
            # fore_data = pd.read_csv('FINAL_FORE_DATA.csv')
            fore_data = pd.read_csv(fileList[0])

            fore_data['ANA_DATE'] = pd.to_datetime(fore_data['ANA_DATE'])
            fore_data['DATE_TIME_KST'] = pd.to_datetime(fore_data['DATE_TIME_KST'])
            fore_data['DATE_TIME'] = fore_data['DATE_TIME_KST'] - dtKst

            fore_data['hour'] = fore_data['ANA_DATE'].dt.strftime("%H")
            fore_data = fore_data.loc[
                (fore_data['hour'] == '00')
                ].reset_index(drop=True)

            # fore_data['hourDiff'] = (fore_data['DATE_TIME'] - fore_data['ANA_DATE']) / pd.Timedelta(hours=1)
            # fore_data = fore_data.loc[
            #     (fore_data['hourDiff'] <= 5)
            #     ].reset_index(drop=True)

            # 분석 시간을 기준으로 0~5시간 내의 예보 시간 사용
            fore_data['hourDiff'] = (fore_data['DATE_TIME'] - fore_data['ANA_DATE']) / pd.Timedelta(hours=1)
            # fore_data = fore_data.loc[
            #     (fore_data['hourDiff'] <= 5)
            #     ].reset_index(drop=True)

            # fore_data.columns

            # 원본 데이터 : 15,319,449
            # orgData = fore_data.merge(pw_data, on=['SRV', 'MESURE_DATE_TM'])
            # orgData = fore_data.merge(pw_data, on=['SRV', 'MESURE_DATE_TM'])
            # raw_data = fore_data.merge(pw_data, on=['SRV', 'MESURE_DATE_TM'])
            # raw_data = fore_data.merge(pw_data, on=['SRV', 'DATE_TIME_KST'])
            raw_data = fore_data.merge(pw_data, how='left', left_on=['SRV', 'DATE_TIME_KST'], right_on=['SRV', 'MESURE_DATE_TM'])

            # raw_data.columns
            # orgData.columns

            # ==============================================================================
            # 단기 예측
            # ==============================================================================
            CNSMR_NO_list = list(set(raw_data['CNSMR_NO']))
            RMSE_list = []
            # MAPE_list = []

            for i, CNSMR_NO in enumerate(CNSMR_NO_list):

                # CNSMR_NO = 'SE0300104001404'
                # CNSMR_NO = 'SE0300111002104'

                saveFile = '{}/{}/model/short_term/{}.model'.format(globalVar['inpPath'], serviceName, CNSMR_NO)
                if (not os.path.exists(saveFile)): continue

                log.info("[CHECK] CNSMR_NO : {} / {}".format(CNSMR_NO, round(i / len(CNSMR_NO_list) * 100.0, 2)))
                # short_term_predict(raw_data, CNSMR_NO)

                # data = raw_data.loc[raw_data['CNSMR_NO'] == CNSMR_NO, ['SRV', 'MESURE_DATE_TM', 'CA_TOT', 'HM', 'PA', 'TA', 'TD', 'WD', 'WS', 'SZA', 'AZA', 'ET', 'TURB', 'GHI_CLR', 'DNI_CLR', 'DHI_CLR', 'SWR', 'SG_PWRER_USE_AM']].reset_index(drop=True)
                data = raw_data.loc[
                    raw_data['CNSMR_NO'] == CNSMR_NO
                    ].reset_index(drop=True)

                data['MESURE_DATE_TM'] = pd.to_datetime(data['MESURE_DATE_TM'])
                data['is_weekday'] = data['MESURE_DATE_TM'].map(check_weekend)
                data['is_spring'] = data['MESURE_DATE_TM'].map(check_spring)
                data['is_summer'] = data['MESURE_DATE_TM'].map(check_summer)
                data['is_fall'] = data['MESURE_DATE_TM'].map(check_fall)
                data['is_winter'] = data['MESURE_DATE_TM'].map(check_winter)

                data['is_weekday'] = data['is_weekday'].astype('category')
                data['is_spring'] = data['is_spring'].astype('category')
                data['is_summer'] = data['is_summer'].astype('category')
                data['is_fall'] = data['is_fall'].astype('category')
                data['is_winter'] = data['is_winter'].astype('category')

                ### 단기 예측
                t = 48

                # 중복 제거
                idxData = data.groupby(['DATE_TIME_KST'])['hourDiff'].idxmin()
                dataL1 = data.iloc[idxData.values].reset_index(drop=True)

                # 자동 머신러닝
                # xCol = ['CA_TOT', 'HM', 'PA', 'TA', 'TD', 'WD', 'WS', 'SZA', 'AZA', 'ET', 'TURB', 'GHI_CLR', 'DNI_CLR', 'DHI_CLR', 'SWR', 'is_weekday', 'is_spring', 'is_summer', 'is_fall', 'is_winter']
                xCol = ['CA_TOT', 'HM', 'PA', 'TA', 'TD', 'WD', 'WS', 'SZA', 'AZA', 'ET', 'TURB', 'GHI_CLR', 'DNI_CLR', 'DHI_CLR', 'SWR', 'is_weekday', 'is_spring', 'is_summer', 'is_fall', 'is_winter', 'CO', 'PM10', 'PM25']
                # xCol = ['CA_TOT', 'HM', 'PA', 'TA', 'TD', 'WD', 'WS', 'SZA', 'AZA', 'ET', 'SWR', 'is_weekday', 'is_spring', 'is_summer', 'is_fall', 'is_winter']
                # xCol = ['CA_TOT', 'HM', 'PA', 'TA', 'TD', 'WD', 'WS', 'SZA', 'AZA', 'ET', 'TURB', 'GHI_CLR', 'DNI_CLR', 'DHI_CLR', 'SWR']
                yCol = 'SG_PWRER_USE_AM'

                trainData = dataL1[:-t].reset_index(drop=True)
                testData = dataL1[-t:].reset_index(drop=True)

                prdData = testData

                # plt.plot(testData['DATE_TIME_KST'], testData['TA'])
                # plt.plot(testData['DATE_TIME_KST'], testData['SG_PWRER_USE_AM'])
                # plt.plot(testData['DATE_TIME_KST'], testData['CO'])
                # plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%H'))
                # plt.show()

                sysOpt['mlModel'].update(
                    {
                        'srvId': CNSMR_NO
                        , 'modelKey': 'AI-FOR-20221101'
                        # , 'isOverWrite': False
                        , 'isOverWrite': True
                    }
                )

                # resPycaret = makeMlModel(sysOpt['mlModel'], xCol, yCol, trainData)
                # log.info('[CHECK] resPycaret : {}'.format(resPycaret))
                #
                # pycaretModel = resPycaret['mlModel']
                # prdData['pycaret'] = pycaretModel.predict(prdData)

                # 머신러닝 불러오기
                # resFlaml = makeFlamlModel(sysOpt['mlModel'], xCol, yCol, trainData)
                resFlaml = makeFlamlModel(sysOpt['mlModel'], xCol, yCol, trainData, testData)
                log.info('[CHECK] resFlaml : {}'.format(resFlaml))

                # 머신러닝 예측
                flamlModel = resFlaml['mlModel']
                prdData['flaml'] = flamlModel.predict(prdData)

                import optuna.integration.lightgbm as lgb

                lgbTrainData = lgb.Dataset(trainData[xCol], trainData[yCol])
                lgbTestData = lgb.Dataset(testData[xCol], testData[yCol], reference=lgbTrainData)

                # best_params, history = {}, []

                params = {
                    'objective' : 'regression'
                    , 'metric' : 'rmse'
                    , 'verbosity' : -1
                    , 'n_jobs' : -1
                }

                # model = lgb.train(params=params, train_set=dtrain, num_boost_round=5000, early_stopping_rounds=5, valid_sets=[dtrain, deval], verbose_eval=False)
                # model = lgb.train(params=params, train_set=dtrain, num_boost_round=5000, early_stopping_rounds=50, valid_sets=[dtrain, deval], verbose_eval=False)
                # model = lgb.train(params=params, train_set=dtrain, num_boost_round=10000, early_stopping_rounds=50, valid_sets=[dtrain, deval], verbose_eval=False)
                model = lgb.train(params=params, train_set=lgbTrainData, num_boost_round=10000, early_stopping_rounds=100, valid_sets=[lgbTrainData, lgbTestData], verbose_eval=False)

                saveModel = '{}/{}/{}-{}-{}-{}-{}.model'.format(globalVar['outPath'], 'AI-FOR-20221101', serviceName, CNSMR_NO, 'final', 'lightgbm', 'for')
                os.makedirs(os.path.dirname(saveModel), exist_ok=True)
                log.info('[CHECK] saveFile : {}'.format(saveModel))

                # if (not os.path.exists(saveModel)):
                #     pickle.dump(model, open(saveModel, 'wb'))
                pickle.dump(model, open(saveModel, 'wb'))

                model = pickle.load(open(saveModel, 'rb'))
                prdData['ML3'] = model.predict(data=testData[xCol])

                # 모델 영향도
                # mainTitle = '{}-{}_{}'.format(prdData['SRV'].iloc[0], CNSMR_NO, 'flaml')
                # saveImg = '{}/{}/{}.png'.format(globalVar['figPath'], sysOpt['mlModel']['modelKey'], mainTitle)
                # featData = pd.DataFrame([flamlModel.feature_names_in_, flamlModel.feature_importances_], index=['key', 'val']).transpose().sort_values(by=['val'], ascending=True)
                # plt.barh(featData['key'], featData['val'])
                # os.makedirs(os.path.dirname(saveImg), exist_ok=True)
                # plt.title(mainTitle)
                # plt.legend()
                # plt.tight_layout()
                # plt.savefig(saveImg, dpi=600, bbox_inches='tight')
                # plt.show()
                # plt.close()

                # mainTitle = '{}-{}_{}'.format(prdData['SRV'].iloc[0], CNSMR_NO, 'lightgbm')
                # saveImg = '{}/{}/{}.png'.format(globalVar['figPath'], sysOpt['mlModel']['modelKey'], mainTitle)
                # plot_importance(model)
                # plt.title(mainTitle)
                # plt.tight_layout()
                # plt.savefig(saveImg, dpi=600, bbox_inches='tight')
                # plt.show()
                # plt.close()

                mainTitle = '[48시간 단기] [{0:s}-{1:s}]'.format(prdData['SRV'].iloc[0], CNSMR_NO)
                saveImg = '{}/{}/{}.png'.format(globalVar['figPath'], serviceName, mainTitle)
                os.makedirs(os.path.dirname(saveImg), exist_ok=True)
                log.info('[CHECK] saveImg : {}'.format(saveImg))

                plt.plot(prdData['DATE_TIME_KST'], prdData['SG_PWRER_USE_AM'], marker='o', label='실측')
                # plt.plot(prdData['MESURE_DATE_TM'], prdData['prd'], label='예측 (RMSE : {:.2f}, {:.2f}%)'.format(
                #     mean_squared_error(prdData['SG_PWRER_USE_AM'], prdData['prd'], squared=False)
                #     , (mean_squared_error(prdData['SG_PWRER_USE_AM'], prdData['prd'], squared=False) / np.nanmean(prdData['SG_PWRER_USE_AM'])) * 100.0)
                #     )
                plt.plot(prdData['DATE_TIME_KST'], prdData['flaml'], label='예측 (RMSE : {:.2f}, {:.2f}%)'.format(
                    mean_squared_error(prdData['SG_PWRER_USE_AM'], prdData['flaml'], squared=False)
                    , (mean_squared_error(prdData['SG_PWRER_USE_AM'], prdData['flaml'], squared=False) / np.nanmean(prdData['SG_PWRER_USE_AM'])) * 100.0)
                    )
                plt.plot(prdData['DATE_TIME_KST'], prdData['ML3'], label='예측2 (RMSE : {:.2f}, {:.2f}%)'.format(
                    mean_squared_error(prdData['SG_PWRER_USE_AM'], prdData['ML3'], squared=False)
                    , (mean_squared_error(prdData['SG_PWRER_USE_AM'], prdData['ML3'], squared=False) / np.nanmean(prdData['SG_PWRER_USE_AM'])) * 100.0)
                    )
                # plt.plot(prdData['MESURE_DATE_TM'], prdData['ML4'], label='예측4 (RMSE : {:.2f}, {:.2f}%)'.format(
                #     mean_squared_error(prdData['SG_PWRER_USE_AM'], prdData['ML4'], squared=False)
                #     , (mean_squared_error(prdData['SG_PWRER_USE_AM'], prdData['ML4'], squared=False) / np.nanmean(prdData['SG_PWRER_USE_AM'])) * 100.0)
                #     )
                # plt.plot(prdData['MESURE_DATE_TM'], prdData['lazy'], label='예측 (lazy) : {:.3f}'.format(round(np.sqrt(mean_squared_error(prdData['SG_PWRER_USE_AM'], prdData['flaml'])), 2)))
                # plt.plot(prdData['MESURE_DATE_TM'], prdData['pycaret'], label='예측 (pycaret) : {:.3f}'.format(round(np.sqrt(mean_squared_error(prdData['SG_PWRER_USE_AM'], prdData['pycaret'])), 2)))

                plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%Y.%m.%d %H'))
                plt.gca().xaxis.set_major_locator(mdates.HourLocator(interval=3))
                plt.gcf().autofmt_xdate()
                # plt.xticks(rotation=45, ha='right')
                plt.xticks(rotation=45)
                plt.title(mainTitle)
                plt.legend()
                plt.tight_layout()
                plt.savefig(saveImg, dpi=600, bbox_inches='tight', transparent=True)
                # plt.show()
                plt.close()

        except Exception as e:
            log.error("Exception : {}".format(e))
            raise e

        finally:
            log.info('[END] {}'.format("exec"))

# ================================================
# 3. 주 프로그램
# ================================================
if __name__ == '__main__':

    print('[START] {}'.format("main"))

    try:

        # 파이썬 실행 시 전달인자를 초기 환경변수 설정
        inParams = {}

        print("[CHECK] inParams : {}".format(inParams))

        # 부 프로그램 호출
        subDtaProcess = DtaProcess(inParams)

        subDtaProcess.exec()

    except Exception as e:
        print(traceback.format_exc())
        sys.exit(1)

    finally:
        print('[END] {}'.format("main"))


'''
Implementation taken from https://arxiv.org/abs/1903.09650
which follows derivation given in https://people.maths.ox.ac.uk/gilesm/files/NA-08-01.pdf
'''
import torch

def safe_inverse(x, epsilon=1E-12):
    return x/(x**2 + epsilon)

class SVDGESDD(torch.autograd.Function):
    @staticmethod
    def forward(self, A):
        U, S, V = torch.svd(A)
        self.save_for_backward(U, S, V)
        return U, S, V

    @staticmethod
    def backward(self, dU, dS, dV):
        U, S, V = self.saved_tensors
        Vt = V.t()
        Ut = U.t()
        M = U.size(0)
        N = V.size(0)
        NS = len(S)

        F = (S - S[:, None])
        F = safe_inverse(F)
        F.diagonal().fill_(0)

        G = (S + S[:, None])
        G.diagonal().fill_(float('inf'))
        G = 1/G

        UdU = Ut @ dU
        VdV = Vt @ dV

        Su = (F+G)*(UdU-UdU.t())/2
        Sv = (F-G)*(VdV-VdV.t())/2

        dA = U @ (Su + Sv + torch.diag(dS)) @ Vt
        if (M>NS):
            dA = dA + (torch.eye(M, dtype=dU.dtype, device=dU.device) - U@Ut) @ (dU/S) @ Vt
        if (N>NS):
            dA = dA + (U/S) @ dV.t() @ (torch.eye(N, dtype=dU.dtype, device=dU.device) - V@Vt)
        return dA

def test_SVDSYMEIG_random():
    M, N = 50, 40
    input = torch.rand(M, N, dtype=torch.float64, requires_grad=True)
    assert(torch.autograd.gradcheck(SVDGESDD.apply, input, eps=1e-6, atol=1e-4))

if __name__=='__main__':
    import os
    import sys
    sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))

    test_SVDSYMEIG_random()


#! /usr/bin/env python
# coding=utf-8
"""
Author: Deean
Date: 2022-10-23 23:51
FileName: LCP/LCP 66. 最小展台数量.py
Description:
"""
from typing import List

class Solution:
    def minNumBooths(self, demand: List[str]) -> int:
        alphabet = [0] * 26
        for word in demand:
            cur = [0] * 26
            for c in word:
                cur[ord(c) - 97] += 1
            for i in range(26):
                alphabet[i] = max(alphabet[i], cur[i])
        return sum(alphabet)

if __name__ == '__main__':
    solution = Solution().minNumBooths(["acd", "bed", "accd"])
    print(solution)
    solution = Solution().minNumBooths(["abc", "ab", "ac", "b"])
    print(solution)


from django.contrib import admin
from django.urls import path,include

urlpatterns = [
    path('admin/', admin.site.urls),
    path('auth_user/', include('auth_user.urls')),
    path('investments/', include('investments.urls')),
    path('stocks/', include('stocks.urls')),
]


import sys
import sqlite3
from PyQt5 import uic, QtWidgets  # Импортируем uic
from PyQt5.QtWidgets import QApplication, QMainWindow, QTableWidgetItem

class Clients(QMainWindow):
    def __init__(self, ab, id_client='', e_type='', type=''):
        super().__init__()
        self.ab = ab
        self.e_type = e_type
        self.type = type
        self.id_client = id_client
        uic.loadUi('forms/client_add.ui', self)
        self.connection = sqlite3.connect("db/library_db.sqlite")
        self.btn_ok.clicked.connect(self.add_client)
        self.btn_cancel.clicked.connect(self.close)

        if self.e_type == 1:
            cursor = self.connection.cursor()
            authors = cursor.execute(f'SELECT name_client FROM clients WHERE id_client = "{self.id_client}"').fetchall()
            self.client_name.setText(str(authors[0][0]))

    def add_client(self):
        if self.e_type == 0:
            cursor = self.connection.cursor()
            cursor.execute(f'INSERT INTO clients(name_client) VALUES ("{self.client_name.text()}")')
            self.connection.commit()
            if not self.type:
                self.ab.tab_add()
            elif self.type == 1:
                self.ab.client_view()
        elif self.e_type == 1:
            if self.id_client:
                cursor = self.connection.cursor()
                res = f'UPDATE clients SET name_client = "{self.client_name.text()}" WHERE id_client = {self.id_client}'
                cursor.execute(res)
                self.connection.commit()
                self.ab.client_view()
            else:
                pass

        self.close()

def except_hook(cls, exception, traceback):
    sys.__excepthook__(cls, exception, traceback)

if __name__ == '__main__':
    app = QApplication(sys.argv)
    ex = Clients()
    ex.show()
    sys.excepthook = except_hook
    sys.exit(app.exec_())


import auth
import devices
import interfaces
import flow_analysis

if __name__ == "__main__":

    input("Premere Invio per autenticarsi tramite le credenziali salvate nel file di configurazione")
    # Ottiene il token
    token = auth.getAuthToken()

    # Stampa il token in console
    print("Token Ottenuto: ", token)

    # Ottiene la lista completa dei device di rete
    input("Premere Invio per ottenere la lista dei device")
    devicesList = devices.getDevicesList(token)
    devices.printDeviceList(devicesList)

    # Trova un device di rete per IP
    ipInput = input("Inserisci l'IP del device da cercare: ")
    device = devices.getDeviceByIP(token, ipInput)
    devices.printDeviceList(device)
    id = device['response'][0]['id']
    print("L'ID del device cercato è: ", id)

    # Trova un device di rete per MAC
    mac_input = input("Inserisci il MAC del device da cercare: ")
    device = devices.getDeviceByIP(token, mac_input)
    devices.printDeviceList(device)
    id = device['response'][0]['id']
    print("L'ID del device cercato è: ", id)

    # Ottiene la lista delle interfacce del device appena cercato
    input("Premere Invio per ottenere la lista delle interfacce del device cercato qui sopra")
    interfacesList = interfaces.getDeviceInterfaces(token, id)
    interfaces.printInterfaceInfo(interfacesList)

    input("Premere Invio per eseguire Flow Analysis tra due indirizzi IP di test")
    source_ip = "10.10.22.98"
    destination_ip = "10.10.22.114"
    # source_ip = input("Inserire l'IP Sorgente: ")
    # destination_ip = input("Inserire l'IP Destinazione: ")
    print("IP Sorgente: ", source_ip)
    print("IP Destinazione: ", destination_ip)

    # Esegue la Flow Analysis tra i due indirizzi
    flow = flow_analysis.runFlowAnalysis(token, source_ip, destination_ip)

    # Stampa i dettagli dell'analisi
    flow_analysis.printFlowDetails(flow, token)

    print("\n\nTrial Version, Demo terminata!")


"""views for the movies"""

from django.shortcuts import render, get_object_or_404
from django.core.paginator import Paginator

from movies.models import Movie, Actor, Category
from .forms import ProductSearch

def display_all(request):
    """views that displays the whole database"""
    form = ProductSearch()
    all_movie = Movie.objects.all().order_by('id_code')
    paginator = Paginator(all_movie, 10)
    page_number = request.GET.get('page')
    page_obj = paginator.get_page(page_number)
    return render(request, 'movies/index.html', {'page_obj': page_obj, 'form':form})

def lire(request, id):
    """views that display details of film selected"""
    movie = get_object_or_404(Movie.objects.filter(id_code=id))
    return render(request, 'movies/lire.html',
                  {'movie': movie})

def home(request):
    """views that displays the home page"""
    form = ProductSearch(request.POST or None)
    context = {'form': form}
    return render(request, 'movies/home.html', context)

def search(request):
    """views that deals with the movie search"""
    if not request.method == 'POST' and 'page' in request.GET:
        if 'search' in request.session:
            request.POST = request.session['search']
            request.method = 'POST'
        form = ProductSearch()
    if request.method == 'POST':
        form = ProductSearch(request.POST or None)
        request.session['search'] = request.POST
        if form.is_valid():
            prod = form.cleaned_data['search']
            list_movie_cat = Category.objects.search_movie_genre(prod)
            search_sub = Movie.objects.search_movie_title(prod)
            list_movie_by_actor = Actor.objects.search_movie_actor(prod)
            list_movie_cat = Category.objects.search_movie_genre(prod)
            if list_movie_by_actor:
                list_movie_actor = Movie.objects.filter(actor__in=list_movie_by_actor).order_by('actor')
                paginator_1 = Paginator(list_movie_actor, 10)
                page_number = request.GET.get('page')
                page_obj = paginator_1.get_page(page_number)
                return render(request, 'movies/index.html', {'page_obj': page_obj, 'form': form})
            if list_movie_cat:
                list_movie_by_cat = Movie.objects.filter(category__in=list_movie_cat).order_by('category')
                paginator = Paginator(list_movie_by_cat, 10)
                page_number = request.GET.get('page')
                page_obj = paginator.get_page(page_number)
                return render(request, 'movies/index.html', {'page_obj': page_obj, 'form': form})
            if search_sub:
                paginator = Paginator(search_sub, 10)
                page_number = request.GET.get('page')
                page_obj = paginator.get_page(page_number)
                return render(request, 'movies/index.html', {'page_obj': page_obj, 'form': form})
            return render(request, 'movies/no_response.html')

    else:
        form = ProductSearch()
        return render(request, 'movies/search.html', {'form': form})

def mention(request):
    """views that displays the mention template"""
    form = ProductSearch(request.POST or None)
    context = {'form': form}
    return render(request, "movies/mentions_legales.html", context)


import redis
import os

def getTemperature():
    # Redis 연결 정보
    redis_host = "127.0.0.1"
    redis_port = 6379
    redis_db = 0

    # Redis 클라이언트 생성
    redis_client = redis.StrictRedis(host=redis_host, port=redis_port, db=redis_db)
    # Get all keys in Redis
    all_keys = redis_client.keys("*")
    result = {}

    for key in all_keys:
        data = redis_client.hgetall(key)
        timestamp = key.decode()
        result[timestamp] = {
            field.decode(): value.decode() for field, value in data.items()
        }

    return result

if __name__ == "__main__":
    getTemperature()


from ppadb.client import Client
import pyautogui

adb = Client(host='127.0.0.1', port=5037)
devices = adb.devices()
if len(devices) == 0:
    print('no device attached')
    quit()
device = devices[0]
#print(device)

def get_img(filename, dir='images/'):
    return dir + filename

def take_screenshot(device):
    image = device.screencap()
    #image = device.shell("screencap -p | sed 's/\r$//' > screen.png")
    with open(get_img('screen.png'), 'wb') as f:
        f.write(image)

#main_button_area = [247,1294,395,146]
#rankup_area = [187,686,530,167]
#gameover_area = [89,997,735,405]
take_screenshot(device)
img = pyautogui.locate(get_img('ok.png'), get_img('screen.png'), confidence=0.8)

print('Area:',img)
#print('Count:',len(img))
#print('Center:',pyautogui.center(img))


from airflow import DAG
from airflow.utils.dates import days_ago
from airflow.models import Variable
from airflow.operators.python_operator import PythonOperator

from datetime import datetime, timedelta
import pyodbc
import pandas as pd
from sqlalchemy import create_engine
import urllib
import requests
import time

        ##########################################################################
        # ------------------------- ДОП. ПАРАМЕТРЫ  ---------------------------- #
        ##########################################################################

## Лимит создания лидов в одном цикле
leadlimit = 9999 ##Пока без ограничений

## URL для Post запроса генерации лида в CRM
url_post = 'https://corp.synergy.ru/api/api_d_crm/crm_create_lead'

## Токен авторизации для CRM
token = Variable.get("CRM_API_Token")

## Формируем header для post запроса
header = {'App-Token': f'{token}'}

## Запрос для отбора локальной очереди на генерацию лидов в рамках одного цикла.
## Далее отработанные данные отметим в NM_api_minion с меткой IS_SENT = 1
query_lead_list = f'''
select top({leadlimit})
dd.*
from dbo.[NM_api_minion] dd
left join dbo.NM_api_minion_hst hst with(nolock) on hst.UNTI_ID = dd.UNTI_ID
where 1=1
and dd.IS_SENT = 0    --Статус "Не отправлен"
and hst.UNTI_ID is null --Проверяем что лид с таким UNTI не отправляли
'''

        ##########################################################################
        # --------------------------- Подключения ------------------------------ #
        ##########################################################################

## Подключение к BIDB01
def db_conn():
    print('db_conn starting')
    # Драйвер для подключения к БД
    driver = 'Driver={ODBC Driver 18 for SQL Server}'+ \
            ';TrustServerCertificate=Yes'+ \
            ';Server=' + 'MSK1-BIDB01.synergy.local' + \
            ';Database=' + 'Analytic' + \
            f';UID={Variable.get("tuz_login")}' +  \
            f';PWD={Variable.get("tuz_password")}'

    params = urllib.parse.quote_plus(driver)
    engine = create_engine(f'mssql+pyodbc:///?odbc_connect={params}')
    return engine

## Получить текущую очередь для генерации
def get_query():
    print('get_query starting')
    ## Выгружаем локальную очередь
    df_lead_list = pd.read_sql(query_lead_list, con=db_conn())
    print('get_query df_lead_list success')
    return df_lead_list

## Получить количество строк локальной очереди
def count_query():
    ## Считаем локальную очередь
    print('count_query start')
    count_lead_list = get_query().loc[get_query()['IS_SENT'] == 0].shape[0]
    print('count_query count_lead_list success')

    return count_lead_list

## Функция для обновления поля IS_SENT в очереди
def update_results(result, unti, lead, contact):
    print(f'update_results start -- result:{result} unti:{unti} lead:{lead} contact:{contact}')
    ## Если получили ответ "Успешно" записываем 1 в таблицу очереди
    if result == 1:
        print('result = 1 start update')
        update_query = f"""
        update	dbo.[NM_api_minion]
        set		IS_SENT = 1,
                CONTACT_ID = {contact},
                LEAD_ID = {lead}
        from	dbo.[NM_api_minion]
        where	UNTI_ID = {unti};
        """

    ## Если получили ответ "Ошибки" записываем 2 в таблицу очереди
    else:
        print('result = else start update')
        update_query = f"""
        update	dbo.[NM_api_minion]
        set		IS_SENT = 2
        from	dbo.[NM_api_minion]
        where	UNTI_ID = {unti};
        """
    ## Создаем подключение для обновления
    print('Connecting to BIDB01')
    conn = pyodbc.connect('Driver={ODBC Driver 18 for SQL Server}'+ \
            ';TrustServerCertificate=Yes'+ \
            ';Server=' + 'MSK1-BIDB01.synergy.local' + \
            ';Database=' + 'Analytic' + \
            f';UID={Variable.get("tuz_login")}' +  \
            f';PWD={Variable.get("tuz_password")}')

    # Открываем подключение
    cursor = conn.cursor()
    print('Connection success')

    # Выполняем обновление
    print('start cursor execute update_query')
    cursor.execute(update_query)

    # Коммитим транзакцию
    conn.commit()

    # Закрываем курсор и подключение
    cursor.close()
    conn.close()

def insert_results(unti, json_value):
    ## Запрос для инсерта в таблицу истории обновления
    print('start insert_query')
    insert_query = f'''
    select
    d.UNTI_ID
    ,'{json_value}' api
    ,d.IS_SENT
    ,d.[VERSION] [CP_TYPE]
    ,d.CONTACT_ID
    ,d.LEAD_ID
    ,GETDATE() SYSMOMENT
    from dbo.NM_api_minion d with(nolock)
    left join NM_api_minion_hst dh with(nolock) on d.UNTI_ID = dh.UNTI_ID
    where
    d.IS_SENT <> 0
    and dh.UNTI_ID is null
    and d.UNTI_ID = {unti}
    '''
    print('try inserting')
    try:
        print('get insert df')
        df_insert = pd.read_sql(insert_query,con=db_conn())
        print('start inserting')
        df_insert.to_sql('NM_api_minion_hst',con=db_conn(),if_exists='append',index=False)
        print('inserting success')
    except:
        print('value already exists in hst table')

                                ###############################################################
                                ###################     ГЕНЕРАТОР ЛИДОВ     ###################
                                ###############################################################

def leadgenerator(leadlimit, url_post, header, query_lead_list):

    print('формируем json df_json_fields')
    ## Оставляем только нужные поля для отправки d JSON
    df_json_fields = get_query()[['LAND', 'VERSION', 'CAMPAIGN' ,'QUOTE_ID'
                ,'UNTI_ID' ,'LEAD_NAME' ,'PHONE' ,'EMAIL' ,'CITY' ,'REGION' ,'COMMENT','GENDER_STUD' ,'BIRTHDATE_STUD'
                ,'DOC_TYPE_STUD' ,'CITIZENSHIP_STUD','SNILS_STUD' ,'PASS_SERIES_STUD','PASS_NUMBER_STUD','PASS_AUTHORITY_STUD','PASS_DATE_STUD','PASS_CODE_STUD'
                ,'LAST_NAME_PARENT','NAME_PARENT','SECOND_NAME_PARENT','BIRTHDATE_PARENT','PHONE_PARENT','EMAIL_PARENT'
                ,'PASS_SERIES_PARENT','PASS_NUMBER_PARENT','PASS_AUTHORITY_PARENT','PASS_DATE_PARENT' ,'PASS_CODE_PARENT' ,'GENDER_PARENT', 'UF_CRM_1456812806']]

    count = count_query()

    print(f'''
        *********************
            count query: {count}
        *********************
        ''')

    while count > 0:
        try:
            for index, row in df_json_fields.iterrows():
                print('*' * 50)
                ## Формируем json файл для каждой строки
                json_value = row.to_json()

                ## Записываем unti_id в качестве переменной
                unti_id = row['UNTI_ID']

                ## Отправляем post запрос на генерацию лидов
                print('try Post request')
                try:
                    res = requests.post(url=url_post, headers=header, data=json_value)
                    print('Post request success')
                except:
                    print('Post request failed')

                ## Записываем результаты в переменные для дальнейшего обновления
                json_result = res.json().get('result')
                json_id_contact = res.json().get('id_contact')
                json_id_lead = res.json().get('id_lead')

                ## Делаем обновления боевой таблицы очереди
                print('start update_results')
                update_results(json_result, unti_id, json_id_lead, json_id_contact)

                ## Записываем данные в таблицу истории
                print('start insert_results')
                insert_results(unti_id, json_value)

                # Таймаут между циклами 1 сек
                print('start timeout')
                time.sleep(5)

                count = count_query()
                print(f'''
                *********************
                    count query: {count}
                *********************
                ''')

        except:
            print('FAIL LOOP')
            break

    print('finish')

##########################################################################
# --------------------------- Настройка DAG ------------------------------ #
##########################################################################

default_args = {'owner': 'Nesh_Markoski',
                'depends_on_past': False,
                'retries': 0,
                'retry_delay': timedelta(seconds=120)
                }

with DAG(dag_id="University_2035_LeadGenerator",
        tags = ['CF/CP'],
        description = "Генерация лидов в CRM системе" ,
        schedule_interval='45 6 * * *',
        start_date=days_ago(1),
        catchup=False,
        default_args=default_args) as dag:

    leadgenerator = PythonOperator(
                    task_id='leadgenerator',
                    python_callable=leadgenerator,
                    op_kwargs={'leadlimit': leadlimit,
                              'url_post':url_post,
                              'header':header,
                              'query_lead_list':query_lead_list})

    leadgenerator


from datetime import datetime, timedelta
import os
from airflow import DAG
from airflow.operators.postgres_operator import PostgresOperator
from airflow.operators.dummy_operator import DummyOperator
from airflow.operators import (StageToRedshiftOperator, LoadFactOperator,
                                LoadDimensionOperator, DataQualityOperator)
from helpers import SqlQueries

#
# The following DAG performs the following functions:
#
#       1. Loads songs and events data from S3 to RedShift into their staging tables, repectively
#       2. Transform these tables into fact and dim tables in RedShift
#       3. Performs a data quality check on the fact table in RedShift
#

default_args = {
    'owner': 'Qing',
    'depends_on_past': False,
    'retries': 3,
    'retry_delay': timedelta(minutes=5),
    'catchup': False,
    'email_on_retry': False,
    'start_date': datetime(2018, 11, 1),
    'end_date': datetime(2018, 11, 2),
}

dag = DAG('udac_qing_dag',
          default_args=default_args,
          description='Load and transform data in Redshift with Airflow',
          schedule_interval='0 0 1 * *'
        )

start_operator = DummyOperator(task_id='Begin_execution',  dag=dag)

#
# The following code will load events data from S3 to RedShift. Using this s3_key
#       allows as to do some backfilling based on the time (year and month) partition
#       the manifest file makes sure that data is copying correctly
#

stage_events_to_redshift = StageToRedshiftOperator(
    task_id='Stage_events',
    redshift_conn_id="redshift",
    aws_credentials_id="aws_credentials",
    table="staging_events",
    s3_bucket="udacity-dend",
    s3_key="log_data/{execution_date.year}/{execution_date.month}",
    file_format="JSON",
    manifest_file="s3://udacity-dend/log_json_path.json",
    dag=dag
)

#
# The following code will load songs data from S3 to RedShift. It
#      provides some extra parameters to accelecrate the process
#

stage_songs_to_redshift = StageToRedshiftOperator(
    task_id='Stage_songs',
    redshift_conn_id="redshift",
    aws_credentials_id="aws_credentials",
    table="staging_songs",
    s3_bucket="udacity-dend",
    s3_key="song_data",
    file_format="JSON",
    manifest_file="auto",
    extra_params="compupdate off acceptinvchars",
    dag=dag
)

#
# The following code will load the fact table from the staging tables
#      Note that

load_songplays_table = LoadFactOperator(
    task_id='Load_songplays_fact_table',
    redshift_conn_id="redshift",
    table="songplays",
    sql=SqlQueries.songplay_table_insert,
    dag=dag
)

#
# The following code will load the dim tables.
#      Note that it allows a 'truncate' function to clean the table before loading
#

load_user_dimension_table = LoadDimensionOperator(
    task_id='Load_user_dim_table',
    redshift_conn_id="redshift",
    table="users",
    sql=SqlQueries.user_table_insert,
    truncate=True,
    dag=dag
)

load_song_dimension_table = LoadDimensionOperator(
    task_id='Load_song_dim_table',
    redshift_conn_id="redshift",
    table="songs",
    sql=SqlQueries.song_table_insert,
    truncate=True,
    dag=dag
)

load_artist_dimension_table = LoadDimensionOperator(
    task_id='Load_artist_dim_table',
    redshift_conn_id="redshift",
    table="artists",
    sql=SqlQueries.artist_table_insert,
    truncate=True,
    dag=dag
)

load_time_dimension_table = LoadDimensionOperator(
    task_id='Load_time_dim_table',
    redshift_conn_id="redshift",
    table="time",
    sql=SqlQueries.time_table_insert,
    truncate=True,
    dag=dag
)

#
# Data quality check on the fact table, i.e., songplays
#

run_quality_checks = DataQualityOperator(
    task_id='Run_data_quality_checks',
    redshift_conn_id="redshift",
    table="songplays",
    columns=['playid', 'start_time', 'userid', 'songid', 'artistid'],
    dag=dag
)

end_operator = DummyOperator(task_id='Stop_execution',  dag=dag)

#
# Task ordering for the DAG tasks
#

start_operator >> stage_events_to_redshift
start_operator >> stage_songs_to_redshift
stage_events_to_redshift >> load_songplays_table
stage_songs_to_redshift >> load_songplays_table
load_songplays_table >> load_user_dimension_table
load_songplays_table >> load_song_dimension_table
load_songplays_table >> load_artist_dimension_table
load_songplays_table >> load_time_dimension_table
load_user_dimension_table >> run_quality_checks
load_song_dimension_table >> run_quality_checks
load_artist_dimension_table >> run_quality_checks
load_time_dimension_table >> run_quality_checks
run_quality_checks >> end_operator