import re, sys, logging, argparse, traceback, ntpath
import matplotlib.pyplot as plt
import json, os, glob, subprocess
import numpy as np
import pandas as pd
import importlib.machinery
from pprint import pformat
from src.utils.logging.logging_segue import create_logger
from src.utils.video.multilevel_video_factory import MultilevelVideoFactory
from src.consts.video_configs_consts import *
from src.consts.manifest_representation_consts import *
from src.consts.splitter_consts import *
from src.consts.augmenter_consts import *
from src.utils.video.level import Level
from src.utils.video_factory import Video, FullVideo, EXTENSION

def pmkdir(kdir):
    if not os.path.exists(kdir):
        os.makedirs(kdir)

def read_configs(configs):
    with open(configs, 'r') as fin:
        data = json.load(fin)
    return data

def empty(folder):
    for filename in os.listdir(folder):
        file_path = os.path.join(folder, filename)
        try:
            if os.path.isfile(file_path) or os.path.islink(file_path):
                os.unlink(file_path)
            elif os.path.isdir(file_path):
                shutil.rmtree(file_path)
        except Exception as e:
            print("Error in empyting log folder")
            sys.exit(-1)

class SimulationFileHandler:
    def __init__(self, args):
        ## parameter check

        assert os.path.exists(args.video_configs), "Video configuration file doesn't exist"
        print(args.segments_handler_configs)
        assert os.path.exists(args.segments_handler_configs), "Grouper configuration file doesn't exist"

        assert os.path.exists(args.segments_structure_in), "Video RAW segments do not exists"

        self.verbose = args.verbose

        self.logs_dir = args.logs_dir
        if os.path.exists(args.logs_dir):
            empty(args.logs_dir)
        pmkdir(args.logs_dir)

        log_file = os.path.join(args.logs_dir, 'make_simulation_file.log')
        self.logger = create_logger('Make Simulation File: Main', log_file, verbose=args.verbose)

        self.logger.info("Simulation file handler module initialized")
        self.logger.info("Logs file stored in {}".format(log_file))

        ## Outputs:
        self.simulation_file_out = args.simulation_file_out
        self.logger.info("Simulation file stored in {}".format(self.simulation_file_out))
        pmkdir(os.path.dirname(self.simulation_file_out))

        self.csv_std_segments_in = pd.read_csv(args.csv_std_segments_in, keep_default_na=False)
        try:
            self.csv_aug_segments_in = pd.read_csv(args.csv_aug_segments_in)
        except:
            self.logger.warning("aug chunks {} does not exist! Continuing without augmentation".format(args.csv_aug_segments_in))
            self.csv_aug_segments_in = None

        self.logger.info("Reference segments csv in: {}".format(args.csv_std_segments_in))
        self.logger.info("Augmented segments csv in: {}".format(args.csv_aug_segments_in))

        self.csv_std_segments_out = args.csv_std_segments_out
        self.logger.info("CSV of standard segments will be stored in {}".format(self.csv_std_segments_out))
        pmkdir(os.path.dirname(self.csv_std_segments_out))

        if args.csv_aug_segments_out:
            self.csv_aug_segments_out = args.csv_aug_segments_out
            self.logger.info("CSV of augmented segments will be stored in {}".format(self.csv_aug_segments_out))

        ## creation of figsdir
        pmkdir(args.figs_dir)
        self.figs_dir = args.figs_dir
        self.logger.info("Figs will be stored in {}".format(self.figs_dir))

        self.video_data = read_configs(args.video_configs)
        self.logger.info("Accounted video data stored = {}".format(args.video_configs))
        self.logger.debug("Accounted video data {}".format(self.video_data))

        self.segments_handler_data = read_configs(args.segments_handler_configs)
        self.logger.info("Accounted augmenter data stored = {}".format(self.segments_handler_data))
        self.logger.debug("Accounted augmenter data {}".format(self.segments_handler_data))

        self.simulation_file_data = read_configs(args.sim_file_configs)
        self.logger.info("Accounted simulation file data stored = {}".format(args.sim_file_configs))
        self.logger.debug("Accounted simulation file data {}".format(self.simulation_file_data))

        self.segments_structure_in = read_configs(args.segments_structure_in)
        self.logger.info("Accounted segments boundaries stored = {}".format(args.segments_structure_in))

        video_original_path = self.video_data[K_VIDEO_PATH]
        assert os.path.exists(video_original_path), "Original video does not exists"

        self.logger.debug("Creating raw video from {}".format(video_original_path))
        self.raw_video  = FullVideo(Video(video_original_path, self.logs_dir, verbose=self.verbose))

        self.logger.info("Raw video from {} created succesfully".format(video_original_path))
        self.logger.debug("Video {} has a total of {} frames for {} fps".format(self.raw_video.video().video_path(),
                                                                                self.raw_video.video().load_total_frames(),
                                                                                self.raw_video.video().load_fps()))
        self.resolutions = self.video_data[K_RESOLUTIONS].split()
        self.logger.debug("Original video resolution = {}".format(self.resolutions))

        self.rescaled_video_template = args.rescaled_video_template
        self.logger.debug("Rescaled video template is {}".format(self.rescaled_video_template))

        self.multires_video = None

        self.rescaled_videos = []
        for res in self.resolutions:
            video_path = self.rescaled_video_template.format(res)
            self.logger.debug("Looking for {}".format(video_path))
            assert os.path.exists(video_path), video_path
            self.rescaled_videos.append(FullVideo(Video(video_path, self.logs_dir, verbose=self.verbose)))
            self.logger.info("Video {} found".format(video_path))

        splitting_module = self.simulation_file_data[K_SPLITTING_MODULE].replace('/','.').replace('.py', '')
        splitting_class = self.simulation_file_data[K_SPLITTING_CLASS]
        self.logger.info("Splitting module = {}".format(splitting_module))
        self.logger.info("Splitting class = {}".format(splitting_class))

        try:
            splitting_module_args = self.simulation_file_data[K_SPLITTING_MODULE_ARGS]
            self.logger.debug("Splitting module args = {}".format(splitting_module_args))
        except:
            splitting_module_args = {}
            self.logger.warning("No splitting module argument has been passed")

        try:
            self.logger.debug("Retrieving splitting policy")
            SplittingPolicy = getattr(importlib.import_module(splitting_module), splitting_class)
            self.logger.info("Splitting policy retrieved correctly")
        except:
            self.logger.error("{} doesn't contain class name {}, or some errors are present in module".format(splitting_module, splitting_class))
            self.logger.exception("message")
            sys.exit(-1)

        try:
            self.logger.debug("Instantiating splitting policy")
            self.splitting_policy_std = SplittingPolicy(   self.raw_video,
                                                           self.rescaled_videos,
                                                           os.path.dirname(self.rescaled_video_template),
                                                           self.segments_structure_in,
                                                           splitting_module_args,
                                                           args.verbose,
                                                           args.logs_dir,
                                                           args.figs_dir)

            self.logger.info("Splitting policy instantiated correctly")
        except:
            self.logger.error("Something went wrong in the instantiation of the class")
            self.logger.exception("message")
            sys.exit(-1)

        self.augmented_videos = []
        self.splitting_policy_aug = None
        try:
            assert self.csv_aug_segments_in is not None
            augmenter_module_args = self.segments_handler_data[K_AUGMENTER_MODULE_ARGS]
            self.logger.debug("Augmenter module args = {}".format(augmenter_module_args))

            augmenter_encoding_name =  augmenter_module_args[K_AUGMENTER_ENCODING_NAME]
            self.logger.info("Looking for augmented segments in {}".format(augmenter_encoding_name))

            for res in self.resolutions:
                video_path = args.augmented_video_template.format(augmenter_encoding_name, res, EXTENSION)
                self.logger.debug("Looking for {}".format(video_path))
                if not os.path.exists(video_path):
                    self.logger.warning("{} does not exists. Skipping..".format(video_path))
                    continue

                self.augmented_videos_template = os.path.dirname(args.augmented_video_template.format(augmenter_encoding_name, '{}', EXTENSION))
                self.augmented_videos.append(FullVideo(Video(video_path, self.logs_dir, verbose=self.verbose)))
                self.logger.info("Video {} found".format(video_path))
            try:
                self.splitting_policy_aug = SplittingPolicy(   self.raw_video,
                                                               self.augmented_videos,
                                                               self.augmented_videos_template,
                                                               self.segments_structure_in,
                                                               splitting_module_args,
                                                               args.verbose,
                                                               args.logs_dir,
                                                               args.figs_dir)
            except:
                self.logger.exception("message")
                sys.exit(-1)
        except:
            self.augmenter_policy = None
            self.logger.info("No augmentation policy selected. Processing video unaugmented")

    def compute_splitting(self):
        self.logger.info("Starting video splitting and computation: std segments")
        self.splitting_policy_std.split_and_compute()
        self.logger.info("Video splitting computed succesfully")
        fact = MultilevelVideoFactory(self.logger, enable_logging=self.verbose)
        self.multires_video = fact.multilevel_video_from_full_videos(self.rescaled_videos)

        # check if there have been some removal #

        remove_map = {}
        self.logger.info("Checking if std segments present removal")

        for i, row in self.csv_std_segments_in.iterrows():
            for res in self.resolutions:
                if BITRATE_CSV_STD.format(res) not in row.keys() or row[BITRATE_CSV_STD.format(res)] == '':
                    if i not in remove_map.keys():
                        remove_map[i] = []
                    remove_map[i].append(self.multires_video.get_std_level(res, i))

        self.multires_video = self.multires_video.remove_levels(remove_map)
        if self.splitting_policy_aug:
            self.logger.info("Starting video splitting and computation: aug segments")
            self.splitting_policy_aug.split_and_compute()

    def compute_augmentation(self):

        assert self.multires_video

        if self.augmented_videos:
            self.logger.info("Adding augmented levels to the video")
            augmented_map = {}
            augmented_videos_map = {}

            for video in self.augmented_videos:
                augmented_videos_map[video.video().load_resolution()] = video

            for row_index, row in self.csv_aug_segments_in.iterrows():
                level = augmented_videos_map[ row[RESOLUTION_CSV]].load_segments()[row[INDEX_CSV]]
                level = Level([level], is_augmented=True)
                if row[INDEX_CSV] not in augmented_map.keys():
                    augmented_map[row[INDEX_CSV]] = []
                augmented_map[row[INDEX_CSV]].append(level)

            self.multires_video = self.multires_video.add_levels(augmented_map)

        else:
            self.logger.info("augmentation policy is None")

    def log_results(self):

        std_dataframe = self.multires_video.load_std_dataframe()
        aug_dataframe = self.multires_video.load_aug_dataframe()
        sim_data = self.multires_video.get_simulation_data()

        std_dataframe.to_csv(self.csv_std_segments_out)
        if aug_dataframe is not None:
            aug_dataframe.to_csv(self.csv_aug_segments_out)

        with open(self.simulation_file_out, 'w') as fout:
            json.dump(sim_data, fout)

if __name__=="__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--video_configs', help='video configurations', required=True)
    parser.add_argument('--segments_handler_configs', help='configurations of the rescaler and aug module', required=True)
    parser.add_argument('--sim_file_configs', help='configurations of the rescaler and aug module', required=True)
    parser.add_argument('--segments_structure_in', help='indicator for the splitting', required=True)
    parser.add_argument('--logs_dir', help='where the logs are gonna be stored', required=True)
    parser.add_argument('--figs_dir', help='where the figs are gonna be stored', required=True)
    parser.add_argument('--simulation_file_out', help='output: json file for the simulation', required=True)
    parser.add_argument('--csv_std_segments_in', help='output: recap csv for segments std', required=True)
    parser.add_argument('--csv_aug_segments_in', help='output: recap csv for segments aug')
    parser.add_argument('--csv_std_segments_out', help='output: recap csv for segments std', required=True)
    parser.add_argument('--csv_aug_segments_out', help='output: recap csv for segments aug')
    parser.add_argument('--rescaled_video_template', help='rescaled video template', required=True)
    parser.add_argument('--augmented_video_template', help='rescaled video template', required=True)
    parser.add_argument('--verbose', action="store_true")

    args = parser.parse_args()
    args.rescaled_video_template = args.rescaled_video_template.format('{}', EXTENSION)
    ra = SimulationFileHandler(args)
    ra.compute_splitting()
    ra.compute_augmentation()
    ra.log_results()


import cv2
import os

def save_frame_camera_key(device_num, dir_path, rfid, ext='jpg', delay=1, window_name='frame'):
    WIDTH = 1920
    HEIGHT = 1080
    FPS = 30
    cap = cv2.VideoCapture(device_num)
    cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'))
    # cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc('Y','U','Y','V'))
    cap.set(cv2.CAP_PROP_FRAME_WIDTH, WIDTH)
    cap.set(cv2.CAP_PROP_FRAME_HEIGHT, HEIGHT)
    cap.set(cv2.CAP_PROP_FPS, FPS)

    if not cap.isOpened():
        return

    os.makedirs(dir_path, exist_ok=True)
    base_path = os.path.join(dir_path, rfid)

    while True:
        ret, frame = cap.read()
        cv2.imshow(window_name, frame)
        key = cv2.waitKey(delay) & 0xFF
        if key == ord('c'):
            cv2.imwrite('{}.{}'.format(base_path, ext), frame)
            return '{}.{}'.format(base_path, ext)
        elif key == ord('q'):
            break

    cv2.destroyWindow(window_name)

ret = save_frame_camera_key(1, 'data/temp', 'aaaaa')
print(ret)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
@author: dj
@contact: dj@itmojun.com
@software: PyCharm
@file: main.py
@time: 2018/12/5 20:32
"""

import time
import urllib, json, random
from flask import Flask, request, abort
from werkzeug.contrib.fixers import ProxyFix
import redis
from wechatpy.utils import check_signature
from wechatpy.exceptions import InvalidSignatureException
from wechatpy import parse_message, create_reply

app = Flask(__name__)
app.wsgi_app = ProxyFix(app.wsgi_app)

pool = redis.ConnectionPool(host='localhost', port=6379, decode_responses=True)

def get_robot_reply(input_text):
    apiKeys = ('cc8c863cfa2b42ecb1e6ae9d4f2c5f36', '512a756fbb5b45b4902e53e09a89e86d', '263aa1a1b0a6470d84ad28415e7bee47', '2d7072e72426447585a3bfc5f72fdc38', 'f7954cb3d7b14134962d0a918185339a')

    data = {
        "reqType":0,
        "perception": {
            "inputText": {
                "text": input_text
            },
        },
        "userInfo": {
            "apiKey": random.choice(apiKeys),
            "userId": "339745"
        }
    }

    data = json.dumps(data, ensure_ascii=False).encode("utf-8")
    url = urllib.request.Request("http://openapi.tuling123.com/openapi/api/v2", data=data, method="POST")
    res = urllib.request.urlopen(url).read()
    return json.loads(res.decode("utf-8"))["results"][0]["values"]["text"]

@app.route('/wx', methods=['GET', 'POST'])
def weixin_handle():
    token = 'itmojun'  # 注意此处要与微信公众平台上填写的保持一致
    timestamp = request.args.get('timestamp', '')
    nonce = request.args.get('nonce', '')
    echo_str = request.args.get('echostr', '')
    signature = request.args.get('signature', '')

    try:
        check_signature(token, signature, timestamp, nonce)
    except InvalidSignatureException:
        # 处理异常情况或忽略
        abort(403)  # 后续代码不会被执行

    if request.method == 'GET':
        # 微信公众号后台设置界面提交服务器URL验证
        return echo_str
        # return reply, 200, {'Content-Type': 'text/plain; charset=utf-8'}
    else:
        # 微信官方服务器通过POST方式转发消息给我们自己的服务器
        msg = parse_message(request.data)

        r = redis.Redis(connection_pool=pool)
        redis_key = msg.source

        if msg.type == 'event':  # 事件消息
            if msg.event == 'subscribe':  # 关注事件
                subscribe_time = msg.create_time.strftime("%Y-%m-%d %H:%M:%S")
                ret = r.hmset(redis_key, {"state": 1, "subscribe_time": subscribe_time})
                if ret:
                    pass  # redis操作成功
                else:
                    pass  # redis操作失败

                reply = create_reply('Hello，小伙伴儿，欢迎关注IT魔君！一入IT深似海，从此再也不缺爱，IT魔君与你分享各种骚操作，带你装逼带你飞！', msg);
            elif msg.event == 'unsubscribe':  # 取消关注事件
                r.delete(redis_key)
                reply = create_reply('期待与你再次相遇！', msg)
            else:
                # 比如点击菜单事件等
                reply = create_reply('', msg)  # 回复空消息
        elif msg.type == 'text' or msg.type == 'voice':  # 文本消息或语音消息
            cur_msg_time = time.time()
            last_msg_time = r.hget(redis_key, 'last_msg_time')  # hget方法返回值为str值或者None
            if not last_msg_time:
                last_msg_time = 0
            ret = r.hset(redis_key, "last_msg_time", cur_msg_time)

            # 如果两次连续消息的时间间隔超过1小时，那么就直接设置为普通聊天模式
            if cur_msg_time - float(last_msg_time) > 3600:
                r.hmset(redis_key, {'state': 1})
                state = '1'

            if msg.type == 'voice':
                content = msg.recognition
                if content is None:
                    reply = create_reply('你的普通话不够标准，请再说一次或打字吧！', msg)
                    return reply.render()
            else:
                content = msg.content

            state = r.hget(redis_key, 'state')  # hget方法返回值为str值或者None
            if not state:
                r.hmset(redis_key, {'state': 1})
                state = '1'

            if state == '1':
                # 普通聊天模式
                if '急急如律令' in content:
                    if r.hexists(redis_key, 'pc_id'):
                        reply = '成功进入远控模式！(已绑定目标电脑ID为“%s”，如需更换绑定，请输入“绑定电脑”)' % r.hget(redis_key, 'pc_id')
                        ret = r.hset(redis_key, "state", 2)
                    else:
                        reply = '成功进入远控模式，请输入要绑定的目标电脑ID'
                        ret = r.hset(redis_key, "state", 21)
                # elif '小魔仙你好' in content:
                elif '@小魔仙' == content:
                        reply = '小魔仙驾到，我们开始快乐地尬聊吧~'
                        ret = r.hset(redis_key, "state", 3)
                else:
                        # reply = ''  # 回复空消息
                        reply = '君哥正在苦逼调bug，稍后给你回复！'
            elif state == '2':
                # 远控模式
                if '芝麻关门' in content:
                    reply = '已退出远控模式，进入与君哥尬聊的快乐模式！'
                    ret = r.hset(redis_key, "state", 1)
                elif '绑定电脑' in content:
                    reply = '目标电脑ID是啥？'
                    ret = r.hset(redis_key, "state", 21)
                else:
                    pc_id = r.hget(redis_key, 'pc_id')
                    cmd = content
                    ret = r.set(pc_id, cmd, ex=3)  # 过期时间为3秒
                    if ret:
                        reply = '控制成功！'
                    else:
                        reply = '控制失败！'
            elif state == '21':
                # 等待用户输入目标电脑ID模式
                pc_id = content
                ret = r.hmset(redis_key, {'state': 2, 'pc_id': pc_id})
                if ret:
                    reply = '绑定成功！'
                else:
                    reply = '绑定失败！'
            elif state == '3':
                # 自动聊天模式
                # if '小魔仙再见' in content:
                if '@君哥' == content:
                    reply = '小魔仙睡觉去了，现在君哥陪你尬聊！'
                    ret = r.hset(redis_key, "state", 1)
                else:
                    try:
                        reply = get_robot_reply(content)
                    except Exception as e:
                        reply = str(e)
            else:
                reply = ''  # 回复空消息

            reply = create_reply(reply, msg)
        else:  # 其他类型消息
            reply = create_reply('很抱歉，我暂时不能处理这种消息！', msg)

        return reply.render()

if __name__ == '__main__':
    app.run(debug=True)


import pandas as pd
import numpy as np
import os
from itertools import product
from scipy.interpolate import interp1d
from datetime import datetime

import netCDF4 as nc
from geopy.distance import distance

from src.constants import MMC
from src.framework import define_equation_elements, define_state_elements

def load_poc_data():

    metadata = pd.read_csv('../../../data/geotraces/poc_conc.csv',
                           usecols=('GTNum', 'GTStn', 'CastType',
                                    'CorrectedMeanDepthm',
                                    'Latitudedegrees_north',
                                    'Longitudedegrees_east',
                                    'DateatMidcastGMTyyyymmdd'))

    # SPM_SPT_pM has NaN for intercal samples, useful for dropping later
    cols = ('SPM_SPT_ugL', 'POC_SPT_uM', 'POC_LPT_uM')

    values = pd.read_csv('../../../data/geotraces/poc_conc.csv', usecols=cols)
    errors = pd.read_csv('../../../data/geotraces/poc_sd.csv', usecols=cols)
    flags = pd.read_csv('../../../data/geotraces/poc_flag.csv', usecols=cols)

    data = merge_poc_data(metadata, values, errors, flags)
    data.dropna(inplace=True)
    data = data.loc[:, ~data.columns.str.startswith('SPM_SPT_ugL')]

    data = data[~data['station'].isin((1, 3, 18.3))]  # exclude stations 1, 18.3
    data = data[data['depth'] < 1000]  # don't need data below 1km

    return data

def merge_poc_data(metadata, values, errors, flags):

    rename_cols = {'GTStn': 'station', 'CastType': 'cast',
                   'CorrectedMeanDepthm': 'depth',
                   'POC_SPT_uM': 'POCS', 'POC_LPT_uM': 'POCL',
                   'Latitudedegrees_north': 'latitude',
                   'Longitudedegrees_east': 'longitude',
                   'DateatMidcastGMTyyyymmdd': 'datetime'}

    for df in (metadata, values, errors, flags):
        df.rename(columns=rename_cols, inplace=True)

    data = pd.merge(metadata, values, left_index=True, right_index=True)
    data = pd.merge(data, errors, left_index=True, right_index=True,
                    suffixes=(None, '_unc'))
    data = pd.merge(data, flags, left_index=True, right_index=True,
                    suffixes=(None, '_flag'))

    return data

def poc_by_station():

    df = load_poc_data()
    data = {}
    maxdepth = 600

    for s in df['station'].unique():
        raw = df[df['station'] == s].copy()
        raw.sort_values('depth', inplace=True, ignore_index=True)
        cleaned = clean_by_flags(raw)
        data[int(s)] = cleaned.loc[cleaned['depth'] < maxdepth]

    return data

def clean_by_flags(raw):

    cleaned = raw.copy()
    flags_to_clean = (3, 4)

    tracers = ('POCS', 'POCL')
    for ((i, row), t) in product(cleaned.iterrows(), tracers):
        if row[f'{t}_flag'] in flags_to_clean:
            poc = cleaned.at[i - 1, t], cleaned.at[i + 1, t]
            depth = cleaned.at[i - 1, 'depth'], cleaned.at[i + 1, 'depth']
            interp = interp1d(depth, poc)
            cleaned.at[i, t] = interp(row['depth'])
            cleaned.at[i, f'{t}_unc'] = cleaned.at[i, t]

    return cleaned

def load_nc_data(dir):

    datainfo = {'modis': {'ext': '.nc', 'dateidx': 3},
                'cbpm': {'ext': '.hdf', 'dateidx': 1}}

    path = f'../../../data/geotraces/{dir}'
    filenames = [f for f in os.listdir(path) if datainfo[dir]['ext'] in f]
    data = {}

    for f in filenames:
        date = f.split('.')[datainfo[dir]['dateidx']]
        if dir == 'cbpm':
            date = datetime.strptime(date, '%Y%j').strftime('%Y%m%d')
        data[date] = nc.Dataset(os.path.join(path, f))

    return data

def extract_nc_data(poc_data, dir):

    var_by_station = {}

    nc_data = load_nc_data(dir)
    nc_dates = [datetime.strptime(d,'%Y%m%d') for d in nc_data]

    if dir == 'cbpm':
        nc_lats = [90 - x*(1/12) - 1/24 for x in range(2160)]
        nc_lons = [x*(1/12) - 180 + 1/24 for x in range(4320)]

    for s in poc_data:

        df = poc_data[s].copy()
        row = df[df['cast'] == 'S'].iloc[0]

        date = datetime.strptime(row['datetime'], '%m/%d/%y %H:%M')
        station_coord = np.array((row['latitude'], row['longitude']))
        prev_nc_dates = [d for d in nc_dates if d <= date]
        nc_date = min(prev_nc_dates, key=lambda x: abs(x - date))
        nc_8day = nc_data[nc_date.strftime('%Y%m%d')]

        if dir == 'cbpm':
            var_name = 'npp'
            var_8day = nc_8day.variables[var_name]
        if dir == 'modis':
            var_name = 'Kd'
            var_8day = nc_8day.variables['MODISA_L3m_KD_8d_4km_2018_Kd_490'][0]
            nc_lats = list(nc_8day.variables['lat'][:])
            nc_lons = list(nc_8day.variables['lon'][:])

        close_nc_lats = [
            l for l in nc_lats if abs(station_coord[0] - l) < 1]
        close_nc_lons = [
            l for l in nc_lons if abs(station_coord[1] - l) < 1]
        nc_coords = list(product(close_nc_lats, close_nc_lons))
        distances = [distance(ncc, station_coord) for ncc in nc_coords]
        nc_coords_sorted = [
            x for _, x in sorted(zip(distances, nc_coords))]

        j = 0
        while True:
            nc_lat_index = nc_lats.index(nc_coords_sorted[j][0])
            nc_lon_index = nc_lons.index(nc_coords_sorted[j][1])
            station_var = var_8day[nc_lat_index, nc_lon_index]
            if station_var > -9999:
                break
            j += 1

        var_by_station[row['station']] = station_var

    return var_by_station

def load_mixed_layer_depths():

    mld_df = pd.read_csv('../../../data/geotraces/mld.csv')
    mld_dict = dict(zip(mld_df['station'], mld_df['depth']))

    return mld_dict

def load_Th_fluxes():

    df = pd.read_csv('../../../data/geotraces/sinkingflux_Th.csv',
                     usecols=('station', 'depth', 'flux'))

    return df

def get_median_POCS():

    poc = poc_by_station()
    data = pd.DataFrame(columns=['depth', 'POCS'])
    for  df in poc.values():
        data = pd.concat([data, df], join='inner', ignore_index=True)

    median = np.median(data['POCS'])

    return median

def get_station_Th_fluxes(grid, max_depth, station, flux_df):

    flux_layers = []
    flux_depths = []
    flux_vals = []
    s_df = flux_df.loc[(flux_df['station'] == station) & (flux_df['depth'] < max_depth)]
    for i, depth in enumerate(grid):
        nearby = s_df.iloc[(s_df['depth'] - depth).abs().argsort()[:1]].iloc[0]
        if nearby['flux'] > 0:
            flux_layers.append(i)
            flux_depths.append(nearby['depth'])
            flux_vals.append(nearby['flux'])
    fluxes = pd.DataFrame(list(zip(flux_layers, flux_depths, flux_vals)), columns=['layer', 'depth', 'flux'])

    return fluxes

def get_station_data(poc_data, params, ez_depths, flux_constraint=False):

    d = {s: {} for s in poc_data}
    mixed_layer_depths = load_mixed_layer_depths()
    if flux_constraint:
        max_depth = 620
        flux_df = load_Th_fluxes()

    for s in poc_data.keys():
        grid = tuple(poc_data[s]['depth'].values)
        layers = tuple(range(len(grid)))
        zg = min(grid, key=lambda x:abs(x - ez_depths[s]))  # grazing depth
        tracers = define_tracers(poc_data[s])
        d[s]['mld'] = mixed_layer_depths[s]
        d[s]['grid'] = grid
        d[s]['latitude'] = poc_data[s].iloc[0]['latitude']
        d[s]['longitude'] = poc_data[s].iloc[0]['longitude']
        d[s]['layers'] = layers
        d[s]['zg'] = zg
        d[s]['ezd'] = ez_depths[s]
        d[s]['umz_start'] = grid.index(zg) + 1
        d[s]['tracers'] = tracers
        if flux_constraint:
            d[s]['Th_fluxes'] = get_station_Th_fluxes(grid, max_depth, s, flux_df)
        else:
            d[s]['Th_fluxes'] = None
        d[s]['e_elements'] = define_equation_elements(tracers, layers, Th_fluxes=d[s]['Th_fluxes'])
        d[s]['s_elements'] = define_state_elements(tracers, params, layers, Th_fluxes=d[s]['Th_fluxes'])

    return d

def define_tracers(data):

    tracers = {'POCS': {}, 'POCL': {}}

    for t in tracers:
        tracers[t]['prior'] = data[t]
        tracers[t]['prior_e'] = data[f'{t}_unc']

    return tracers

def define_residuals(prior_error, gamma):

    residuals = {'POCS': {}, 'POCL': {}}

    for tracer in residuals:
        residuals[tracer]['prior'] = 0
        residuals[tracer]['prior_e'] = gamma * prior_error

    return residuals

def set_param_priors(params, Lp_prior, Po_prior, B3_prior, mc_params):

    def set_prior(param_name, prior, error):

        params[param_name]['prior'] = prior
        params[param_name]['prior_e'] = error

    set_prior('B2p', mc_params['B2p'], mc_params['B2p'])
    set_prior('Bm2', mc_params['Bm2'], mc_params['Bm2'])
    set_prior('Bm1s', mc_params['Bm1s'], mc_params['Bm1s'])
    set_prior('Bm1l', mc_params['Bm1l'], mc_params['Bm1l'])
    set_prior('ws', mc_params['ws'], mc_params['ws'])
    set_prior('wl', mc_params['wl'], mc_params['wl'])
    set_prior('Po', Po_prior, Po_prior*0.5)
    set_prior('Lp', Lp_prior, Lp_prior*0.5)
    set_prior('B3', B3_prior, B3_prior*0.5)
    set_prior('a', 0.3, 0.3*0.5)
    set_prior('zm', 500, 500*0.5)

def define_param_uniformity():

    param_uniformity = {}

    nonuniform_params = ('B2p', 'Bm2', 'Bm1s', 'Bm1l', 'ws', 'wl')
    uniform_params = ('Po', 'Lp', 'B3', 'a', 'zm')

    for p in nonuniform_params:
        param_uniformity[p] = {'dv': True}

    for p in uniform_params:
        param_uniformity[p] = {'dv': False}

    return param_uniformity

def get_Lp_priors(poc_data):

    Kd = extract_nc_data(poc_data, 'modis')
    Lp_priors = {station: 1/k for station, k in Kd.items()}

    return Lp_priors

def get_ez_depths(Lp_priors):

    depths = {station: l*np.log(100) for station, l in Lp_priors.items()}

    return depths

def get_Po_priors(poc_data, Lp_priors, npp_data):
    """Calculate Po priors at each station.

    Args:
        poc_data (_type_): _description_
        Lp_priors (_type_): _description_
        npp_data (_type_): _description_

    Returns:
        _type_: _description_

    Full equation for Po is Po = npp / [Lp * (1 - exp[-ez_depth / Lp])],
    but ez_depth = Lp * ln(100), so (1-exp[]) simplifies to 0.99.
    """
    Po_priors = {s: ((npp_data[s] / MMC)
                     / (Lp_priors[s] * 0.99)) for s in poc_data}

    return Po_priors

def get_B3_priors(npp_data):

    B3_priors = {}

    for s in npp_data:
        B3_priors[s] = 10**(-2.42 + 0.53*np.log10(npp_data[s]))

    return B3_priors


"""
StreamRecorder: resting-state recording
=======================================

A resting-state recording is a simple offline recording during which the brain
activity of a subject is measured in the absence of any stimulus or task. A
resting-state recording can be designed with a `~bsl.StreamRecorder`.
"""

# %%

# Authors: Mathieu Scheltienne <mathieu.scheltienne@fcbg.ch>
#
# License: LGPL-2.1

# sphinx_gallery_thumbnail_path = '_static/stream_recorder/stream_recorder_cli.gif'

# %%
# .. warning::
#
#     Both `~bsl.StreamPlayer` and `~bsl.StreamRecorder` create a new process
#     to stream or record data. On Windows, mutliprocessing suffers a couple of
#     restrictions. The entry-point of a multiprocessing program should be
#     protected with ``if __name__ == '__main__':`` to ensure it can safely
#     import and run the module. More information on the
#     `documentation for multiprocessing on Windows
#     <https://docs.python.org/2/library/multiprocessing.html#windows>`_.
#
# This example will use a sample EEG resting-state dataset that can be retrieve
# with :ref:`bsl.datasets <api/utils:Datasets>`. The dataset is stored in the user home
# directory in the folder ``bsl_data`` (e.g. ``C:\Users\User\bsl_data``).

# %%

import os
import time
from pathlib import Path

import mne

from bsl import StreamPlayer, StreamRecorder, datasets
from bsl.triggers import MockTrigger

# %%
#
# To simulate an actual signal coming from an LSL stream, a `~bsl.StreamPlayer`
# is used with a 40 seconds resting-state recording.

stream_name = "StreamPlayer"
fif_file = datasets.eeg_resting_state.data_path()
player = StreamPlayer(stream_name, fif_file)
player.start()
print(player)

# %%
#
# For this example, the folder ``bsl_data/examples`` located in the user home
# directory will be used to stored recorded files. To ensure its existence,
# `os.makedirs` is used.

record_dir = Path("~/bsl_data/examples").expanduser()
os.makedirs(record_dir, exist_ok=True)
print(record_dir)

# %%
#
# For this simple offline recording, the goal is to start a
# `~bsl.StreamRecorder`, send an event on a trigger to mark the beginning of
# the resting-state recording, wait for a defined duration, and stop the
# recording.
#
# By default, a `~bsl.StreamRecorder` does not require any argument. The
# current working directory is used to record data from all available streams
# in files named based on the date/time timestamp at which the recorder is
# started.
#
# To record only a subset of the available streams with a specific file name
# and in a specific directory, the arguments ``record_dir``, ``fname`` and
# ``stream_name`` must be provided.
#
# For this example, the directory used to store recordings is
# ``bsl_data/examples`` and the file name will start with
# ``example-resting-state``.
#
# .. note::
#
#     By default, the `~bsl.StreamRecorder.start` method is blocking and will
#     wait for the recording to start. This behavior can be changed with the
#     ``blocking`` argument.

recorder = StreamRecorder(record_dir, fname="example-resting-state")
recorder.start()
print(recorder)

# %%
#
# Now that a `~bsl.StreamRecorder` is started and is acquiring data, a trigger
# to mark the beginning of the segment of interest is created. For this
# example, a `~bsl.triggers.MockTrigger` is used, but this example
# would be equally valid with a different type of trigger.

trigger = MockTrigger()

# %%
#
# To mark the beginning of the segment of interest in the recording, a signal
# is sent on the trigger. For this example, the event value (1) is used.

trigger.signal(1)

# %%
#
# Finally, after the appropriate duration, the recording is interrupted.

time.sleep(2)  # 2 seconds duration
del trigger
recorder.stop()
print(recorder)

# %%
#
# A `~bsl.StreamRecorder` records data in ``.pcl`` format. This file can be
# open with `pickle.load`, and is automatically converted to a `~mne.io.Raw`
# FIF file in a subdirectory ``fif``. The recorded files name syntax is:
#
# - If ``fname`` is not provided: ``[date/time timestamp]-[stream]-raw.fif``
# - If ``fname`` is provided: ``[fname]-[stream]-raw.fif``
#
# Where ``stream`` is the name of the recorded LSL stream. Thus, one file is
# created for each stream being recorded.

fname = record_dir / "fif" / "example-resting-state-StreamPlayer-raw.fif"
raw = mne.io.read_raw_fif(fname, preload=True)
print(raw)
events = mne.find_events(raw, stim_channel="TRIGGER")
print(events)

# %%
#
# As for the `~bsl.StreamPlayer`, the `~bsl.StreamRecorder` can be used as a
# context manager. The context manager takes care of starting and stopping the
# recording.

with StreamRecorder(record_dir):
    time.sleep(1)

# %%
#
# As for the `~bsl.StreamPlayer`, the `~bsl.StreamRecorder` can be started via
# command-line when a LSL stream is accessible on the network.
#
# Example assuming:
#
# - the current working directory is ``bsl_data`` in the user home directory
# - the stream to connect to is named ``MyStream``
# - the recorded file naming scheme is ``test-[stream]-raw.fif``, i.e.
#   ``test-MyStream-raw.fif``
#
# .. code-block:: console
#
#     $ bsl_stream_recorder -d examples -f test -s MyStream
#
# .. image:: ../../_static/stream_recorder/stream_recorder_cli.gif
#    :alt: StreamRecorder
#    :align: center

# %%
# | Stop the mock LSL stream used in this example.

player.stop()


# -*- coding: utf-8 -*-
"""
This module allows validation of dictionaries or lists that contain unstructured data.
While using python objects is the right approach to have structured data, many apps
nevertheless use dictionaries or lists of various data and are stuck with it, specifically
for saving configuration etc.
"""
import re
import os
import jinja2

# Import all the types from six to support both PY2 and PY3
from six import integer_types
from six import string_types
from six import text_type
from six import binary_type
from six import iteritems

# TODO: Move all the strings to one place
type_mismatch = 'Expecting value to be {type} but got {actual_type} for {level}'
invalid_boolean = 'Invalid Value for boolean field'

default_template_dir = os.path.join(os.path.dirname(__file__), "doc_templates")

def get_bool(val):
    """
    Convert various True/False values to true boolean values

    :param val: Boolean like value
    :return:
    """
    if val in (0, 'false', 'False', False):
        return False
    elif val in (1, 'true', 'True', True):
        return True
    else:
        raise ValueError(invalid_boolean)

class Schema(object):
    """ Basic class through which data validation can be done.
    Create a schema object by providing schema dictionary as argument.  Once the schema
    is created call realize() to get the normalized schema, and validate to the validate
    a document against the schema.
    >>> s = Schema({'type':'string', 'display_name':'Root'})
    >>> s.validate('xyz')
    []
    >>> s.validate(10)
    ["Expecting value to be (<type 'str'>, <type 'unicode'>) but got <type 'int'> for root(Root)"]
    >>> s.realize()
    {'type': 'string', 'allow_none': False, 'display_name': 'Root', 'description': ''}
    """
    def __init__(self, schema_dict):
        self.root = SchemaNode.create_schema_node('root', schema_dict)

    def validate(self, data):
        return self.root.validate(data)

    def realize(self):
        realized_schema = {}
        self.root.realize_schema(realized_schema)
        return realized_schema

    def document(self, template_directory=None):
        self.realize()

        if template_directory:
            search_path = [template_directory, default_template_dir]
        else:
            search_path = [default_template_dir]
        jinja_env = jinja2.Environment(
            loader=jinja2.FileSystemLoader(search_path),
            autoescape=True
        )
        jinja_env.filters['strip_disp_name'] = lambda x: x.split('(')[0]

        template =  jinja_env.get_template("overall2.html")
        return template.render(root = self.root)

class SchemaNode(object):
    allowed_expansions = {
        ('known_children', dict),
        ('sub_schema', dict),
        ('minimum_value', int),
        ('maximum_value', float),
        ('allowed_values', (list, set, tuple))
    }
    expected_types = None
    type = None

    def __init__(self, level, schema_dict):
        self.level = level
        self._level = level
        try:
            self.display_name = schema_dict.pop('display_name')
            self.description = schema_dict.pop('description', None)
        except:
            raise SchemaError('display_name is mandatory at level %s' % self.level)
        self.level += '(%s)' % self.display_name
        self.realized = False
        self.verbatim = schema_dict.pop('verbatim', None)
        self.allow_none = schema_dict.pop('allow_none', False)
        self.custom_validation = schema_dict.pop('custom_validation', None)

    def process_children(self):
        raise SchemaError('CODE ERROR: Each child node must implement this method')

    def _execute_if_necessary(self, code_like, expected_type):
        if isinstance(code_like, (expected_type, SchemaNode)):
            ret_value = code_like
            executed = False
        else:
            executed = True
            if callable(code_like):
                ret_value = code_like()
            elif isinstance(code_like, basestring):
                try:
                    ret_value = eval(code_like)
                except:
                    raise SchemaError("Guessed the value of (%s) at %s to be code, but evaluation failed" % (code_like, self.level))
            else:
                raise SchemaError("Don't known how to execute dynamic schema at %s (expected type = %s, actual type =%s)" % (self.level,expected_type, type(code_like)))

            if not isinstance(ret_value, expected_type):
                schema_type_mismatch = "Dynamic schema generated %s doesn't match the expected type %s at %s"
                raise SchemaError( schema_type_mismatch % (type(ret_value), expected_type, self.level))

        return ret_value, executed

    def _realize_node(self):
        for key_name, expected_type in self.allowed_expansions:
            obj = getattr(self, key_name, None)
            if obj:
                try:
                    obj, executed = self._execute_if_necessary(obj, expected_type)
                    if executed:
                        setattr(self, key_name, obj)
                except SchemaError:
                    raise
                #except Exception as ex:
                    #raise ex
                    # raise SchemaError("Error (%s) when realizing dynamic schema for %s at %s" % (str(ex), key_name, self.level))

        self.realized = True

    def realize_schema(self, attrs):
        if not self.realized:
            self._realize_node()

        attrs['display_name'] = self.display_name
        if self.description is not None:
            attrs['description'] = self.description
        attrs['type'] = self.type
        attrs['allow_none'] = self.allow_none
        if self.verbatim is not None:
            attrs['verbatim'] = self.verbatim
        if self.custom_validation:
            attrs['custom_validation'] = {}
            attrs['custom_validation']['enabled'] = True
            attrs['custom_validation']['info'] = self.custom_validation.__doc__

    def validate(self, data):
        """
        Validate the data and return the violations found

        :param data:
        :return: List of violations.  Each violation is basically a string, and it is unstructured.
        """
        # Realize if necessary
        if not self.realized:
            self._realize_node()

        if data is None:
            if self.allow_none:
                return [ ]
            else:
                return [
                    "Null is not allowed at level %s" % self.level
                ]

        # Perform common validation
        if not isinstance(data, self.expected_types):
            return [
                type_mismatch.format(type=str(self.expected_types),
                                     actual_type = str(type(data)),
                                     level=self.level)
            ]

        # Perform node specific validation
        schema_errors =  self.validate_data(data)
        if self.custom_validation:
            schema_errors.extend("%s: %s at %s" % (self.custom_validation.__name__, x, self.level) for x in self.custom_validation(data))

        return schema_errors

    def get_target(self):
        return self.level.split('(')[0].replace('.','_')

    def validate_data(self, data):
        raise SchemaError('CODE ERROR: Each child node must implement this method')

    # methods related to documentation
    def get_short_decoration(self):
        return ""

    def should_doc_children(self):
        return False

    def doc_child_list(self):
        return [ ]

    def get_doc_tags(self):
        return {}

    @staticmethod
    def create_schema_node(level, schema_dict):
        # Get the type of the node and create the object
        schema_dict = schema_dict.copy()
        node_type = schema_dict.pop('type', 'map')
        if node_type == 'map':
            schema_node = MapNode(level, schema_dict)
        elif node_type == 'string':
            schema_node = StringNode(level, schema_dict)
        elif node_type == 'number':
            schema_node = NumberNode(level, schema_dict)
        elif node_type == 'list':
            schema_node = ListNode(level, schema_dict)
        elif node_type == 'boolean':
            schema_node = BooleanNode(level, schema_dict)
        elif node_type == 'any':
            schema_node = AnyNode(level, schema_dict)
        else:
            raise SchemaError('Unknown type at level %s' % level)

        # We must have consumed every key in the dictionary
        if len(schema_dict) > 0:
            raise SchemaError('Invalid entries (%s) in schema at level %s for type %s' %
                                                (','.join(schema_dict.keys()), level, node_type))
        # Return the schema node
        return schema_node

class AnyNode(SchemaNode):
    expected_types = (string_types, text_type, list, dict, set, tuple, integer_types, float)

    def validate_data(self, data):
        return [ ]

    def get_short_decoration(self):
        return "*"

    def get_doc_tags(self):
        return {
            "WARNING": "Values set at this level are not validated. Exercise caution."
        }

class StringNode(SchemaNode):
    expected_types = (string_types, text_type)
    type = 'string'

    def __init__(self, level, schema_dict):
        super(StringNode, self).__init__(level, schema_dict)
        self.allowed_values = schema_dict.pop('allowed_values', [])
        pattern = schema_dict.pop('allowed_pattern', None)
        self.valid_pattern = re.compile(pattern) if pattern else None

    def realize_schema(self, attrs):
        super(StringNode, self).realize_schema(attrs)
        if self.allowed_values:
            attrs['allowed_values'] = self.allowed_values
        if self.valid_pattern:
            attrs['allowed_pattern'] = self.valid_pattern

    def get_short_decoration(self):
        return "a"

    def get_doc_tags(self):
        tags = {}
        if self.allowed_values:
            tags['Allowed Values'] = ", ".join(self.allowed_values)
        if self.valid_pattern:
            tags['Allowed Pattern'] = self.valid_pattern
        return tags

    def validate_data(self, data):
        # Check for valid values
        if self.allowed_values and data not in self.allowed_values:
            invalid_value = '%s is not a allowed value for %s. Expect it to be one of: %s'
            return [ invalid_value % (data, self.level, ",".join(self.allowed_values)) ]

        if self.valid_pattern and not self.valid_pattern.match(data):
            return [ "%s does't match expression %s at %s" % (data, self.valid_pattern.pattern, self.level)]

        return [ ]

class SubSchemaNode(SchemaNode):
    subschema_denote = '.n'
    def __init__(self, level, schema_dict):
        super(SubSchemaNode, self).__init__(level, schema_dict)
        self._subschema_realized = False
        self._sub_schema = None
        self.sub_schema = schema_dict.pop('value_schema', None)

    def realize_schema(self, attrs):
        super(SubSchemaNode, self).realize_schema(attrs)
        if self.sub_schema:
            attrs['value_schema'] = { }
            self.sub_schema.realize_schema(attrs['value_schema'])

    def get_sub_schema(self):
        return self._sub_schema

    def set_sub_schema(self, obj):
        if isinstance(obj, dict):
            if self._subschema_realized:
                raise SchemaError('Subschema already realized')

            try:
                self.sub_schema = SchemaNode.create_schema_node(self._level+self.subschema_denote, obj)
            except KeyError:
                raise SchemaError('List type node requires a value_schema at %s' % self.level)

            self._subschema_realized = True
        else:
            self._sub_schema = obj

    sub_schema = property(get_sub_schema, set_sub_schema)

class ListNode(SubSchemaNode):
    expected_types = (list, set, tuple)
    type = 'list'
    subschema_denote = "[i]"

    def __init__(self, level, schema_dict):
        super(ListNode, self).__init__(level, schema_dict)
        self.min_size = schema_dict.pop('minimum_size', None)
        self.max_size = schema_dict.pop('maximum_size', None)

        unique = schema_dict.pop('unique', None)
        if unique is not None:
            self.unique = get_bool(unique)
        else:
            self.unique = None

        if self.min_size and self.max_size and self.min_size > self.max_size:
            raise SchemaError('minimum_size can not be greater than maximum_size at %s' % self.level)

        if not self.sub_schema:
            raise SchemaError('value_schema is mandatory for list type')

    def get_doc_tags(self):
        return {
            'values': 'Must be unique' if self.unique else 'May have duplicates',
            'minimum size': self.min_size,
            'maximum size': self.max_size
        }

    def realize_schema(self, attrs):
        super(ListNode, self).realize_schema(attrs)
        if self.min_size is not None:
            attrs['minimum_size'] = self.min_size
        if self.max_size is not None:
            attrs['maximum_size'] = self.max_size
        if self.unique is not None:
            attrs['unique'] = self.unique

    def should_doc_children(self):
        return True

    def get_short_decoration(self):
        #return r'☰'
        return '[ ]'

    def doc_child_list(self):
        return [ ('N/A', self.sub_schema) ]

    def validate_data(self, data):
        schema_errors = []

        if self.min_size and len(data) < self.min_size:
            schema_errors.append('Minimum size is set to %s, but actual size is %s at level %s' % (self.min_size, len(data), self.level))

        if self.max_size and len(data) < self.min_size:
            schema_errors.append('Maximum size is set to %s, but actual size is %s at level %s' % (self.max_size, len(data), self.level))

        for i, each_value in enumerate(data):
            schema_errors.extend(self.sub_schema.validate(each_value))

        if self.unique:
            dups = set()
            found = set()
            for x in data:
                if x in found:
                    dups.add(x)
                else:
                    found.add(x)
            if dups:
                schema_errors.append('Duplicate(s) %s found for a unique list at %s' % ( ",".join(str(a) for a in dups), self.level))

        return schema_errors

class NumberNode(SchemaNode):
    """ Defines a schema node for numeric data. At present limited to integers only
    """
    expected_types = integer_types
    type = 'number'

    def __init__(self, level, schema_dict):
        super(NumberNode, self).__init__(level, schema_dict)
        self.min_value = schema_dict.pop('minimum_value', None)
        self.max_value = schema_dict.pop('maximum_value', None)

        if self.min_value and self.max_value and self.min_value > self.max_value:
            raise SchemaError("Min and Max values in the schema are incorrect at %s" % self.level)

    def realize_schema(self, attrs):
        super(NumberNode, self).realize_schema(attrs)
        if self.min_value:
            attrs['minimum_value'] = self.min_value
        if self.max_value:
            attrs['maximum_value'] = self.max_value

    def get_doc_tags(self):
        tags = {}
        if self.min_value:
            tags['Minimum Value'] = self.min_value
        if self.max_value:
            tags['Maximum Value'] = self.max_value
        return tags

    def get_short_decoration(self):
        return "1"

    def validate_data(self, data):
        if self.min_value and data < self.min_value:
            return [ "Value %s is smaller than %s at %s" % (data, self.min_value, self.level)]

        if self.max_value and data > self.max_value:
            return [ "Value %s is greater than %s at %s" % (data, self.max_value, self.level)]

        return [ ]

class BooleanNode(SchemaNode):
    """ Defines a schema node for boolean data
    """
    expected_types = bool
    type = 'boolean'

    def __init__(self, level, schema_dict):
        super(BooleanNode, self).__init__(level, schema_dict)
        self.true_value = schema_dict.pop('true_value', 'True')
        self.false_value = schema_dict.pop('false_value', 'False')

    def realize_schema(self, attrs):
        super(BooleanNode, self).realize_schema(attrs)
        attrs['true_value'] = self.true_value
        attrs['false_value'] = self.false_value

    def get_short_decoration(self):
        return r'✓'

    def get_doc_tags(self):
        return {
            'True Value': self.true_value,
            'False Value': self.false_value
        }

    def validate_data(self, data):
        """
        No additional validations necessary for boolean data

        :param data: data
        :return:
        """
        return [ ]

class MapNode(SubSchemaNode):
    """ Defines a schema node for the dictionary data.
    """
    expected_types = dict
    type = 'map'
    subschema_denote = ".<name>"

    def __init__(self, level, schema_dict):
        super(MapNode, self).__init__(level, schema_dict)

        # Create schema nodes for all known children and make sure there is default schema if
        # a name defines no specific schema
        self._preset = False
        self.known_children = schema_dict.pop('known_children', {})
        self.allow_list = schema_dict.pop('allow_list', False)

        if self.allow_list:
            self.expected_types = (dict, list)

        # Check for allow unknown values
        try:
            self.allow_unknown_children = get_bool(schema_dict.pop('allow_unknown_children', False))
            if self.allow_unknown_children and self.sub_schema is None:
                raise SchemaError('Unknown children is true without value schema at %s' % level)
        except ValueError:
            raise SchemaError('Unknown boolean value for allow_unknown_children at %s' % level)

        # Get other attributes
        self.mandatory_names = set(schema_dict.pop('mandatory_children', []))

    def get_doc_tags(self):
        additional_children = [x[0] for x in iteritems(self.known_children) if x[1] == None]
        tags = {
            'Unknown children': 'Allowed' if self.allow_unknown_children else 'Not Allowed',
            'Can be a list': 'Yes' if self.allow_list else 'No'
        }
        if additional_children:
            tags['Additinal Children'] = ", ".join(additional_children)
        if self.mandatory_names:
            tags['Must Provide'] = ",".join(self.mandatory_names)
        return tags

    def realize_schema(self, attrs):
        super(MapNode, self).realize_schema(attrs)

        attrs['known_children'] = {}
        for k, v in iteritems(self.known_children):
            attrs['known_children'][k] = {}
            if v:
                v.realize_schema(attrs['known_children'][k])

        attrs['mandatory_children'] = list(self.mandatory_names)

    def get_short_decoration(self):
        return "{ }"

    def should_doc_children(self):
        return True

    def doc_child_list(self):
        if self.sub_schema:
            yield ('<Name>', self.sub_schema)
        for k,v in iteritems(self.known_children):
            if v:
                yield k, v

    def validate_data(self, data):
        schema_errors = []

        def do_validate(data):
            names_found = set()
            # Go to the next level of validation
            for each_key in data:

                if each_key not in self.known_children and self.allow_unknown_children == False:
                    schema_errors.append('%s is not allowed at level %s' % (each_key, self.level))

                sub_schema = self.known_children.get(each_key) or self.sub_schema
                if not sub_schema:
                    schema_errors.append("No sub-schema found for %s at %s" % (each_key, self.level))
                else:
                    schema_errors.extend(sub_schema.validate(data[each_key]))

                names_found.add(each_key)

            remaining_names = self.mandatory_names - names_found
            if remaining_names:
                schema_errors.append("Values are required for %s at %s" % (",".join(remaining_names), self.level))

        if self.allow_list:
            if isinstance(data, list):
                level_save = self.level
                for i,x in enumerate(data):
                    self.level = level_save + str(i)
                    do_validate(x)
            else:
                do_validate(data)
        else:
            do_validate(data)

        return schema_errors

    def set_known_children(self, child_object):
        if isinstance(child_object, dict):
            if (self._preset):
                raise SchemaError('CODE ERROR: Setting children twice')
            self._preset = True
            self._known_children = {}
            for k,v in iteritems(child_object):
                if v:
                    self._known_children[k] = SchemaNode.create_schema_node(self._level+'.'+k, v)
                else:
                    self._known_children[k] = None
                    if self.sub_schema is None:
                        raise SchemaError('Name %s defines no schema and there is no value schema at %s' % (k, self._level))
        else:
            self._preset = False
            self._known_children = child_object

    def get_known_children(self):
        return self._known_children

    known_children = property(get_known_children, set_known_children)

class SchemaError(Exception):
    """ This is a marker to know SchemaErrors from other errors. No
    additional methods are defined inside.
    """
    pass


"""
Tres de Tres - Partidos

- alimentar: Alimentar los partidos a partir de un archivo CSV
"""
from pathlib import Path
import csv
import click

from lib.safe_string import safe_string

from citas_admin.app import create_app
from citas_admin.extensions import db

from citas_admin.blueprints.tdt_partidos.models import TdtPartido

app = create_app()
db.app = app

@click.group()
def cli():
    """Tres de Tres - Partidos"""

@click.command()
@click.argument("entrada_csv")
def alimentar(entrada_csv):
    """Alimentar a partir de un archivo CSV"""
    ruta = Path(entrada_csv)
    if not ruta.exists():
        click.echo(f"AVISO: {ruta.name} no se encontró.")
        return
    if not ruta.is_file():
        click.echo(f"AVISO: {ruta.name} no es un archivo.")
        return
    click.echo("Alimentando partidos...")
    contador = 0
    with open(ruta, encoding="utf8") as puntero:
        rows = csv.DictReader(puntero)
        for row in rows:
            TdtPartido(
                nombre=safe_string(row["nombre"], to_uppercase=True, save_enie=True),
                siglas=safe_string(row["siglas"], to_uppercase=True, save_enie=True),
                estatus=row["estatus"],
            ).save()
            contador += 1
            if contador % 100 == 0:
                click.echo(f"  Van {contador}...")
    click.echo(f"{contador} partidos alimentados.")

cli.add_command(alimentar)


import os
from pathlib import Path

featureName = input("Please enter Usecase\n")
ext = 'ts'
pathDomain = './src/domain/usecases/{0}'.format(
    featureName[0].lower()+featureName[1:])
pathRepository = './src/data/repository/{0}'.format(
    featureName[0].lower()+featureName[1:])
pathDataSource = './src/data/dataSource/{0}'.format(
    featureName[0].lower()+featureName[1:])

if not os.path.exists(pathDomain):   # create folders if not exists
    Path(pathDomain).mkdir(parents=True, exist_ok=True)

if not os.path.exists(pathRepository):   # create folders if not exists
    Path(pathRepository).mkdir(parents=True, exist_ok=True)

if not os.path.exists(pathDataSource):   # create folders if not exists
    Path(pathDataSource).mkdir(parents=True, exist_ok=True)

useCase = '{0}Usecase'.format(featureName)
useCaseImpl = '{0}UsecaseImpl'.format(featureName)

repository = '{0}Repository'.format(featureName)
repositoryImpl = '{0}RepositoryImpl'.format(featureName)

dataSource = '{0}DataSource'.format(featureName)
localDataSource = '{0}LocalDataSource'.format(featureName)
remoteDataSource = '{0}RemoteDataSource'.format(featureName)

f_useCase = os.path.join(pathDomain, '{0}.{1}'.format(useCase, ext))
f_useCaseImpl = os.path.join(pathDomain, '{0}.{1}'.format(useCaseImpl, ext))

f_dataSource = os.path.join(pathDataSource, '{0}.{1}'.format(dataSource, ext))
f_localDataSource = os.path.join(
    pathDataSource, '{0}.{1}'.format(localDataSource, ext))
f_remoteDataSource = os.path.join(
    pathDataSource, '{0}.{1}'.format(remoteDataSource, ext))

f_repository = os.path.join(pathRepository, '{0}.{1}'.format(repository, ext))
f_repositoryImpl = os.path.join(
    pathRepository, '{0}.{1}'.format(repositoryImpl, ext))

contentUsecase = ("export interface {0} {{}} " +
                  "\nexport class {1} implements {0} {{" +
                  "\nprivate repository: {2}" +
                  "\nconstructor(_repository: {2}) {{" +
                  "\n  this.repository = _repository" +
                  "\n}}" +
                  "\n}} ").format(
    useCase, useCaseImpl, repository)

contentRepository = ("export interface {0} {{}} " +
                     "\nexport class {1} implements {0} {{" +
                     "\n    private localDataSource: {2}" +
                     "\n    private remoteDataSource: {2}" +

                     "\nconstructor(_localDataSource: {2},_remoteDataSource:{2}) {{" +
                     "\n    this.localDataSource = _localDataSource" +
                     "\n    this.remoteDataSource = _remoteDataSource" +

                     "\n}}}}").format(
    repository, repositoryImpl, dataSource)

contentDataSource = "export interface {0} {{}}".format(
    dataSource)

contentLocalDataSource = "export class {0} implements {1} {{}}".format(
    localDataSource, dataSource)

contentRemoteDataSource = "export class {0} implements {1} {{}}".format(
    remoteDataSource, dataSource)

fileUseCase = open(f_useCase, "x")
fileUseCase.write(contentUsecase)
fileUseCase.close()

fileRepository = open(f_repository, "x")
fileRepository.write(contentRepository)
fileRepository.close()

fileDataSource = open(f_dataSource, "x")
fileDataSource.write(contentDataSource)
fileDataSource.close()

fileLocalDataSource = open(f_localDataSource, "x")
fileLocalDataSource.write(contentLocalDataSource)
fileLocalDataSource.close()

fileRemoteDataSource = open(f_remoteDataSource, "x")
fileRemoteDataSource.write(contentRemoteDataSource)
fileRemoteDataSource.close()


import numpy as np
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, BatchNormalization, Activation
from keras.layers.convolutional import Conv2D, MaxPooling2D
from keras.constraints import maxnorm

def Init_Train_Test(outputFile, activation, x_train, y_train, x_test, y_test, seed, class_num, epochs):
    # model
    model = Sequential()

    model.add(Conv2D(32, (3, 3), input_shape=x_train.shape[1:], activation='relu', padding='same'))
    model.add(Dropout(0.2))
    model.add(BatchNormalization())

    model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.2))
    model.add(BatchNormalization())

    model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.2))
    model.add(BatchNormalization())

    model.add(Conv2D(128, (3, 3), padding='same', activation='relu'))
    model.add(Dropout(0.2))
    model.add(BatchNormalization())

    model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.2))
    model.add(BatchNormalization())

    model.add(Conv2D(128, (3, 3), padding='same', activation='relu'))
    model.add(Dropout(0.2))
    model.add(BatchNormalization())

    model.add(Flatten())
    model.add(Dropout(0.2))

    model.add(Dense(256, kernel_constraint=maxnorm(3)))
    model.add(Activation('relu'))
    model.add(Dropout(0.2))
    model.add(BatchNormalization())

    model.add(Dense(128, kernel_constraint=maxnorm(3)))
    model.add(Activation('relu'))
    model.add(Dropout(0.2))
    model.add(BatchNormalization())

    model.add(Dense(class_num))
    model.add(Activation(activation))

    # train
    optimizer = 'adam'
    model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
    np.random.seed(seed)
    model.fit(x_train, y_train, validation_data=(x_test, y_test), epochs=epochs, batch_size=64)

    # test
    scores = model.evaluate(x_test, y_test, verbose=0)
    print("Accuracy: %.2f%%" % (scores[1] * 100))

    # save model
    model.save(outputFile)


import time
import datetime

# below codes will only focus on how to create a file name with current time being a part of file name
# make sure to add codes to goto required working directory
# get the current time information from system

partial_time = str(datetime.datetime.now())
#print local time - for info only
print(partial_time)

logfile_name = 'Test_report_' + partial_time[0:10]+ '_' + partial_time[11:13] + partial_time[14:16]
log_file = logfile_name + '.' + 'txt'
print(log_file)

# this will create a file named as logfile in the current directory ; it is a text file
write_file = open(log_file, 'w')
write_file.close()

""" output:
#this is system time format
2018-05-30 23:37:36.509681

# remove the ':' from 23:37 ; second is not used; file extension added as 'txt'
Test_report_2018-05-30_2337.txt

# this is final name :
Test_report_2018-05-30_2337

"""


""" TensorMONK :: architectures """

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from ..layers import Convolution, ResidualComplex, PrimaryCapsule, \
    RoutingCapsule

class CapsuleNet(nn.Module):
    r"""Dynamic routing between capsules - https://arxiv.org/pdf/1710.09829.pdf
    for MNIST (tensor_size = (1, 1, 28, 28)) -- works for FashionMNIST

    Args:
        tensor_size: shape of tensor in BCHW
            (None/any integer >0, channels, height, width)
        n_labels: 10, MNIST/FashionMNIST
        primary_n_capsules: number of capsules in primary capsule layer
        primary_capsule_length: length of each capsule in primary capsule layer
        routing_capsule_length: number of capsules in routing capsule
        routing_iterations: length of each capsule in routing capsule
        replicate_paper: replicates https://arxiv.org/pdf/1710.09829.pdf
        activation: None/relu/relu6/lklu/elu/prelu/tanh/sigm/maxo/rmxo/swish
        dropout: 0. - 1., default = 0.1 with dropblock=True
        normalization: None/batch/group/instance/layer/pixelwise
        pre_nm: if True, normalization -> activation -> convolution else
            convolution -> normalization -> activation

    Return:
        embedding (a torch.Tensor), rec_tensor (a torch.Tensor),
            rec_loss (a torch.Tensor)
    """
    def __init__(self,
                 tensor_size: tuple = (6, 1, 28, 28),
                 n_labels: int = 10,
                 primary_n_capsules: int = 8,
                 primary_capsule_length: int = 32,
                 routing_capsule_length: int = 16,
                 routing_iterations: int = 3,
                 replicate_paper: bool = True,
                 activation: str = "relu",
                 dropout: float = 0.,
                 normalization: str = None,
                 pre_nm: bool = False,
                 *args, **kwargs):
        super(CapsuleNet, self).__init__()

        if replicate_paper:
            primary_n_capsules = 8
            primary_capsule_length = 32
            routing_capsule_length = 16
            routing_iterations = 3
            block = Convolution
            self.InitialConvolutions = \
                Convolution(tensor_size, filter_size=9, out_channels=256,
                            strides=1, pad=False, activation="relu")
            _tensor_size = self.InitialConvolutions.tensor_size
        else:  # You can be creative!
            block = Convolution
            kwargs = {"activation": activation, "dropout": dropout,
                      "normalization": normalization, "pre_nm": pre_nm}
            tp = [Convolution(tensor_size, 5, 64, 1, False, **kwargs),
                  ResidualComplex((6,  64, 24, 24), 3, 256, 1, True, **kwargs),
                  ResidualComplex((6, 256, 24, 24), 3, 256, 1, True, **kwargs),
                  ResidualComplex((6, 256, 24, 24), 3, 256, 1, True, **kwargs),
                  ResidualComplex((6, 256, 24, 24), 3, 256, 1, True, **kwargs),
                  Convolution((6, 256, 24, 24), 5, 64, 1, False, "", **kwargs)]
            self.InitialConvolutions = nn.Sequential(tp)
            _tensor_size = self.InitialConvolutions[-1].tensor_size
        print("InitialConvolutions output size :: ", _tensor_size)

        # block can be replaced with any module available in NeuralLayers
        self.Primary = PrimaryCapsule(_tensor_size, filter_size=9,
                                      out_channels=256, strides=2,
                                      pad=False, activation="",
                                      dropout=dropout, batch_nm=False,
                                      pre_nm=False, block=block,
                                      n_capsules=primary_n_capsules,
                                      capsule_length=primary_capsule_length)
        print("Primary capsule output size :: ",
              self.Primary.tensor_size)
        self.Routing = RoutingCapsule(self.Primary.tensor_size,
                                      n_capsules=n_labels,
                                      capsule_length=routing_capsule_length,
                                      iterations=routing_iterations)

        print("Routing capsule output size :: ", self.Routing.tensor_size)
        self.Reconstruction = \
            nn.Sequential(nn.Linear(n_labels*routing_capsule_length, 512),
                          nn.ReLU(),
                          nn.Linear(512, 1024),
                          nn.ReLU(),
                          nn.Linear(1024, int(np.prod(tensor_size[1:]))))

        self.tensor_size = (6, n_labels)
        self.input_tensor_size = tensor_size

    def forward(self, tensor, targets):
        tensor_size = tensor.size()

        # CapsuleNet
        tensor_deep = self.InitialConvolutions(tensor)
        tensor_primary = self.Primary(tensor_deep)
        embedding = self.Routing(tensor_primary)

        # Reconstruction (only during training)
        #   remove 'if' loop if you like to view the rec_tensor on test data
        rec_tensor = None
        rec_loss = 0.
        if self.training:
            identity = torch.eye(self.tensor_size[1])
            if targets.is_cuda:
                identity = identity.cuda()
            onehot_targets = identity.index_select(dim=0,
                                                   index=targets.view(-1))

            rec_tensor = (embedding *
                          onehot_targets[:, :, None]).view(tensor_size[0], -1)
            rec_tensor = torch.tanh(self.Reconstruction(rec_tensor))
            rec_loss = F.mse_loss(rec_tensor.view(tensor_size[0], -1),
                                  tensor.view(tensor_size[0], -1))
            rec_tensor = rec_tensor.view(*tensor_size)

        return embedding, rec_tensor, rec_loss

# from tensormonk.layers import *
# tensor_size = (2, 1, 28, 28)
# targets = torch.LongTensor([1, 2])
# tensor = torch.rand(*tensor_size)
# test = CapsuleNet(tensor_size)
# test(tensor, targets)[2]


import pygame
from pygame.locals import *

class GameObject(object):

    animation_swap_time = 75

    def __init__(self):
        self.clips = []
        self.rect  = None
        self.speed = 0
        self.image = None

    def __init__(self, imgPath):
        self.speed = 0
        self.clips = []
        self.image = pygame.image.load(imgPath)
        self.rect  = self.image.get_rect()
        self.current_clip = 0

    def setPos(self, x, y):
        self.rect.left = x
        self.rect.top  = y

    def move(self, xspeed, yspeed):
        self.rect.left += xspeed
        self.rect.top  += yspeed

    def setImage(self, image):
        self.image = image
        self.rect = image.get_rect()
        self.current_clip = -1

    def draw(self):
        if len(self.clips) == 0:
            pygame.display.get_surface().blit(self.image, self.rect)
        else:
            if self.current_clip == 0:
                self.rect.width = self.clips[self.current_clip].width
            self.rect.height = self.clips[self.current_clip].height
            # print self.clips
            pygame.display.get_surface().blit(self.image.subsurface(self.clips[self.current_clip]), self.rect)


from django.views import generic
from django.urls import reverse_lazy
from .models import NewsStory
from .forms import StoryForm

class IndexView(generic.ListView):
    template_name = 'news/index.html'

    def get_queryset(self):
        '''Return all news stories.'''
        return NewsStory.objects.order_by('pub_date')

    def get_context_data(self, **kwargs):
        print(self.request.GET.get('search'))
        search_var = self.request.GET.get('search',"")
        context = super().get_context_data(**kwargs)
        # context['latest_stories'] = NewsStory.objects.order_by('-pub_date')[:3]
        # context['all_stories'] = NewsStory.objects.order_by('-pub_date')[3:100]
        # context['latest_stories'] = NewsStory.objects.order_by('-pub_date').filter(title__contains=search_var)[:3]
        # context['all_stories'] = NewsStory.objects.order_by('-pub_date').filter(title__contains=search_var)[3:]
        context['latest_stories'] = NewsStory.objects.order_by('-pub_date').filter(author__username__contains=search_var)[:3]
        context['all_stories'] = NewsStory.objects.order_by('-pub_date').filter(author__username__contains=search_var)[3:]
        return context

class StoryView(generic.DetailView):
    model = NewsStory
    template_name = 'news/story.html'
    context_object_name = 'story'

class AddStoryView(generic.CreateView):
    form_class = StoryForm #from \news\forms.py
    context_object_name = 'storyForm' #just a name for the object
    template_name = 'news/createStory.html'
    success_url = reverse_lazy('news:index') #the path we can post things to when they are succesful

    #overiding form_valid on generic.CreateView
    def form_valid(self, form):
        #set author to logged in user
        form.instance.author = self.request.user
        return super().form_valid(form)


from playwright.sync_api import sync_playwright
from pathlib import Path
from urllib.parse import urlparse, urlunparse
import time

"""
https://playwright.dev/python/docs/api/class-route
"""
def custom_headers(url) -> dict:
    res = urlparse(url)
    loc = res.netloc
    host = urlunparse([res.scheme, res.netloc, '', '', '', ''])
    page_url = urlunparse([res.scheme, res.netloc, res.path, '', '', ''])
    return {
        'authority': loc,
        'accept': 'application/json, text/javascript, */*; q=0.01',
        'accept-language': 'zh-CN,zh;q=0.9',
        'origin': host,
        'referer': host,
        'sec-ch-ua': '"Google Chrome";v="113", "Chromium";v="113", "Not-A.Brand";v="24"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"macOS"',
        'sec-fetch-dest': 'empty',
        'sec-fetch-mode': 'cors',
        'sec-fetch-site': 'same-site',
        'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36',
        'x-referer-page': page_url,
        'x-rp-client': 'h5_1.0.0',
    }

with sync_playwright() as p:
    url = 'https://item.jd.com/7836786.html#crumb-wrap'
    extra_http_headers = custom_headers(url)
    browser = p.chromium.launch(headless=False)
    context = browser.new_context(extra_http_headers=extra_http_headers)
    page = context.new_page()
    page.goto(url)
    print(context.storage_state())
    time.sleep(5)
    page.screenshot(path='example1.png')
    browser.close()


##############################################################################
# 6.1.7 어느 갱신 방법을 이용할 것인가?
# SGD, 모멘텀, AdaGrad, Adam의 학습 패턴을 비교합니다.
##############################################################################

import sys,os
sys.path.append(os.pardir)  # 부모 디렉터리의 파일을 가져올 수 있도록 설정
import numpy as np
import matplotlib.pyplot as plt
from collections import OrderedDict
from common.optimizer import *

# 함수 f
def f(x, y):
    return x ** 2 / 20.0 + y ** 2

# f의 미분 함수
def df(x, y):
    return x / 10.0, 2.0 * y

init_pos = (-7.0, 2.0)

# 초기화
params = {}
params['x'], params['y'] = 0, 0
grads = {}
grads['x'], grads['y'] = 0, 0

"""Optimezer 별로 학습 패턴 비교"""
optimizers = OrderedDict()
optimizers["SGD"] = SGD(lr=0.95)
optimizers["Momentum"] = Momentum(lr=0.1)
optimizers["AdaGrad"] = AdaGrad(lr=1.5)
optimizers["RMSprop"] = RMSprop(lr=0.2)
# optimizers["Adam"] = Adam(lr=0.3)

# 크게 10번만 가보자
# optimizers["SGD"] = SGD(lr=1.05)
# optimizers["Momentum"] = Momentum(lr=0.8)

# 작게 99번만 가보자
# optimizers["SGD"] = SGD(lr=0.35)
# optimizers["Momentum"] = Momentum(lr=0.033)

# 더 작게...
# optimizers["Momentum 0.01"] = Momentum(lr=0.01)   # 380
# optimizers["Momentum 0.001"] = Momentum(lr=0.001) # 3500

idx = 1
# optimizer 별로 실행해서 비교해보자
for key in optimizers:
    optimizer = optimizers[key]
    x_history = []
    y_history = []
    params['x'], params['y'] = init_pos[0], init_pos[1]  # x와 y 초기값 설정

    # 30회 반복
    for i in range(30):
        x_history.append(params['x'])
        y_history.append(params['y'])

        # 기울기 구하기
        grads['x'], grads['y'] = df(params['x'], params['y'])   # 미분

        # 기울기 최적화 업데이트
        #   - 예)     초기값 세팅된 1회 : params: {'x': -7.0, 'y': 2.0} & grads : {'x': 0, 'y': 0}
        #   - 예) 최적화 업데이트 후 2회 : params: {'x': -5.5, 'y': 0.5} & grads : {'x': -0.7, 'y': 4.0}
        #         ...
        optimizer.update(params, grads)

    # x축과 y축의 값
    x = np.arange(-10, 10, 0.01)  # -10 <= x < 10 (0.01 만큼씩) : x: [-10. - 9.99 - 9.98...   9.97   9.98   9.99]
    y = np.arange(-5, 5, 0.01)    #  -5 <= y <  5 (0.01 만큼씩) : y: [ -5. - 4.99 - 4.98...   4.97   4.98   4.99]

    # x축을 나타내는 점들, y축을 나타내는 점들 => 하나의 그리드
    X, Y = np.meshgrid(x, y)
    # 최적화된 기울기를 함수 f 결과 값
    Z = f(X, Y)

    # 외곽선 단순화
    mask = Z > 7
    Z[mask] = 0  # 결과값 중 7보다 큰 것을(True) 0으로 masking.

    # 그래프 그리기 (2 X 2, )
    plt.subplot(2, 2, idx)
    idx += 1
    plt.plot(x_history, y_history, 'o-', color="red")
    plt.contour(X, Y, Z)    # X, Y, Z 윤곽 표시 (등고선)
    plt.ylim(-10, 10)
    plt.xlim(-10, 10)
    plt.plot(0, 0, '+')  # (0,0) 위치 표시
    # colorbar()
    # spring()
    plt.title(key)
    plt.xlabel("x")
    plt.ylabel("y")

plt.show()


import time

try:
    from typing import Tuple
except ImportError:
    pass

try:
    ticks_diff = time.ticks_diff
    ticks_ms = time.ticks_ms
except AttributeError:
    def ticks_ms():
        return time.time_ns() / 1000000

    def ticks_diff(t1, t2):
        # TODO: Probably need to handle wrap around here.
        return t1 - t2

def _is_leap_year(year: int) -> bool:
    if year % 100 == 0:
        return year % 400 == 0
    return year % 4 == 0

def _days_in_month(year: int, month: int) -> int:
    if month in (4, 6, 9, 11):
        return 30
    if month == 2:
        return 29 if _is_leap_year(year) else 28
    return 31

class Date:
    """Represents a specific date."""
    def __init__(self, year: int, month: int, day: int):
        assert year >= 1970, "Dates before 1970 are not supported."
        assert month >= 1 and month <= 12, f"Month {month} out of range."
        assert day >= 1 and day <= _days_in_month(year, month), \
            f"Day {day} out of range for {year} {month}."
        self.__days_since_1970 = (year - 1970) * 365
        for i in range(1970, year):
            if _is_leap_year(i):
                self.__days_since_1970 += 1
        for m in range(1, month):
            self.__days_since_1970 += _days_in_month(year, m)
        self.__days_since_1970 += day

    def __year_month_day(self) -> Tuple[int, int, int]:
        y, m, d = 1970, 1, self.__days_since_1970
        while d > (366 if _is_leap_year(y) else 365):
            d -= (366 if _is_leap_year(y) else 365)
            y += 1
        while d > _days_in_month(y, m):
            d -= _days_in_month(y, m)
            m += 1
        return y, m, d

    def weekday(self) -> int:
        """Returns the day of the week. 0 is Monday, 6 is Sunday."""
        return ((self.__days_since_1970 % 7) + 2) % 7

    def __iadd__(self, days: int) -> "Date":
        self.__days_since_1970 += days
        return self

    def __isub__(self, days: int) -> "Date":
        self.__days_since_1970 -= days
        return self

    @property
    def year(self) -> int:
        return self.__year_month_day()[0]

    @property
    def month(self) -> int:
        return self.__year_month_day()[1]

    @property
    def day(self) -> int:
        return self.__year_month_day()[2]

    def __eq__(self, other: object) -> bool:
        if not isinstance(other, Date):
            return False
        return self.__days_since_1970 == other.__days_since_1970

class DateTime:
    """Represents a specific date and a time."""
    BASE: int = 0

    def __init__(self,
                 year: int,
                 month: int,
                 day: int,
                 weekday: int = 0,
                 hour: int = 0,
                 minute: int = 0,
                 second: int = 0,
                 microsecond: int = 0) -> None:
        self.__date = Date(year, month, day)
        self.hour = hour
        self.minute = minute
        self.second = second

    def weekday(self) -> int:
        """Returns the day of the week. 0 is Monday, 6 is Sunday."""
        return self.__date.weekday()

    @property
    def year(self) -> int:
        return self.__date.year

    @property
    def month(self) -> int:
        return self.__date.month

    @property
    def day(self) -> int:
        return self.__date.day

    def __add__(self, other: "TimeDelta") -> "DateTime":
        dt = DateTime(self.year,
                      self.month,
                      self.day,
                      hour=self.hour,
                      minute=self.minute,
                      second=self.second)
        dt.second += other.seconds
        while dt.second > 60:
            dt.minute += 1
            dt.second -= 60
        while dt.minute > 60:
            dt.hour += 1
            dt.minute -= 60
        dt.hour += other.hours
        while dt.hour > 24:
            dt.__date += 1
            dt.hour -= 24
        dt.__date += other.days
        return dt

    def __iadd__(self, other: "TimeDelta") -> "DateTime":
        self.second += other.seconds
        while self.second > 60:
            self.minute += 1
            self.second -= 60
        while self.minute > 60:
            self.hour += 1
            self.minute -= 60
        self.hour += other.hours
        while self.hour > 24:
            self.__date += 1
            self.hour -= 24
        self.__date += other.days
        return self

    def __sub__(self, other: "TimeDelta") -> "DateTime":
        dt = DateTime(self.year,
                      self.month,
                      self.day,
                      hour=self.hour,
                      minute=self.minute,
                      second=self.second)
        dt.second -= other.seconds
        while dt.second < 0:
            dt.minute -= 1
            dt.second += 60
        while dt.minute < 0:
            dt.hour -= 1
            dt.minute += 60
        dt.hour -= other.hours
        while dt.hour < 0:
            dt.__date -= 1
            dt.hour += 24
        dt.__date -= other.days
        return dt

    def __isub__(self, other: "TimeDelta") -> "DateTime":
        self.second -= other.seconds
        while self.second < 0:
            self.minute -= 1
            self.second += 60
        while self.minute < 0:
            self.hour -= 1
            self.minute += 60
        self.hour -= other.hours
        while self.hour < 24:
            self.__date -= 1
            self.hour += 24
        self.__date -= other.days
        return self

    def __eq__(self, other: object) -> bool:
        if not isinstance(other, DateTime):
            return False
        return self.__date == other.__date \
            and self.hour == other.hour \
            and self.minute == other.minute \
            and self.second == other.second

    def __str__(self):
        return f"{self.year}-{self.month:02n}-{self.day:02n} " \
             + f"{self.hour:02n}:{self.minute:02n}:{self.second:02n}"

    def __repr__(self):
        return f"{self.year}-{self.month:02n}-{self.day:02n} " \
             + f"{self.hour:02n}:{self.minute:02n}:{self.second:02n}"

class TimeDelta:
    """Represents a difference between two times."""
    def __init__(self,
                 days: int = 0,
                 hours: int = 0,
                 seconds: int = 0) -> None:
        self.days = days
        self.hours = hours
        self.seconds = seconds


from scipy.io import loadmat
import h5py  # tables  # or
import numpy as np
import os
import torch
import pickle

class RenderWithTransportMat():

    def __init__(self, transportMatFname, lightHeight, doHalfMat=True):
        self.transportMatFname = transportMatFname
        self.mat_ImgHeight = lightHeight
        # self.renderHeight = renderHeight

        self.doHalfMat = doHalfMat

        self.matT, self.maskT = self.load()
        # self.renderVectors = None

    def load(self):
        typeMat = 'half' if self.doHalfMat else 'full'
        # print('[Render]loading: %s' % (os.path.split(self.transportMatFname)[1]))
        assert os.path.exists(self.transportMatFname), 'FileNotFound %s' % self.transportMatFname
        ext = os.path.split(self.transportMatFname)[1].split('.')[-1]
        if ext == 'mat':
            try:
                mat = loadmat(self.transportMatFname)
                matT = mat['T'].astype('float32')
                maskT = mat['mask'].astype('bool')
            except NotImplementedError:
                mat = h5py.File(self.transportMatFname, 'r')
                matT = mat.get('T')[:].astype('float32').transpose()
                maskT = mat.get('mask')[:].astype('bool').transpose()
                mat.close()
        elif ext == 'pkl':
            with open(self.transportMatFname, 'rb') as infile:
                mat = pickle.load(infile)
                matT = mat['T'].astype('float32')
                maskT = mat['mask'].astype('bool')
        elif ext == 'msgpack':
            import lz4.frame, msgpack, msgpack_numpy
            from functools import partial
            lz4open = partial(lz4.frame.open, block_size=lz4.frame.BLOCKSIZE_MAX1MB,
                            compression_level=lz4.frame.COMPRESSIONLEVEL_MIN)
            with lz4open(self.transportMatFname, "rb") as infile:
                raw = infile.read()
                mat = msgpack.unpackb(raw, object_hook=msgpack_numpy.decode, max_str_len=2**32-1)
                matT = mat[b'T'].astype('float32')
                maskT = mat[b'mask'].astype('bool')
        # matT.shape == [r*r, eH*eW]
        return matT, maskT

    def rendering(self, ims, dst='vector'):
        if type(ims).__module__.find('numpy') >= 0:
            if self.doHalfMat:
                ims = ims.astype('float32')
                H = ims.shape[1]
                ims = ims[:, 0: H // 2, :, :]
                render_vector = self.renderingNP(ims)
                if dst == 'vector':
                    out = render_vector
                elif dst == 'image':
                    out = self.reshapeNP(render_vector)
            return out
        elif type(ims).__module__.find('torch') >= 0:
            if self.doHalfMat:
                H = self.mat_ImgHeight
                ims = ims[:, 0: H // 2, :, :]
                # print("Ims",ims.shape)
                render_vector = self.renderingTorch(ims)
                if dst == 'vector':
                    out = render_vector
                elif dst == 'image':
                    # print("Render", render_vector.shape)
                    out = self.reshapeTorch(render_vector)
                    out = out.permute(0,3,1,2)
                    # print("Out ", out.shape)
            return out

    def renderingTorch(self, ims):
        matT = torch.Tensor(self.matT).to(ims.device)
        nPixels = matT.shape[0]
        N, H, W, C = ims.shape
        assert H * W == matT.shape[1], '[ERROR] transportMat %d doesn\'t match image size %d' % (H * W, matT.shape[1])

        # operation from matlab, reshape in column order
        ims = ims.permute([0, 2, 1, 3])
        lights = ims.contiguous().view((-1, H * W, C))  # [N, #pixel, C]
        # transpose and reshape
        lightsT = lights.permute([1, 0, 2])  # [W*H, N, C]
        lightsTR = lightsT.contiguous().view([W * H, -1])  # [W*H, N*C]
        vectors = torch.matmul(matT, lightsTR)  # [#pixel, W*H] x [W*H, N*C)] -> [#pixel, N*C]
        vectors = vectors.view([nPixels, -1, C])  # [#pixel, N, C]
        vectors = vectors.permute([1, 0, 2])  # [N, #pixel, C]
        renderVectors = vectors
        return renderVectors

    def _reshapeTorch(self, tl_WHxC):
        # todo batch reshape
        C = tl_WHxC.shape[1]
        img = torch.zeros((self.maskT.shape[0], self.maskT.shape[1], C)).to(tl_WHxC.device)
        # img[torch.Tensor(self.maskT.astype(int)).nonzero()] = tl_WHxC. # non differentiable
        img[torch.Tensor(self.maskT.astype(int)).to(tl_WHxC.device) == 1] = tl_WHxC

        img = img.permute([1, 0, 2])
        return img

    def reshapeTorch(self, renderVectors):
        N = renderVectors.shape[0]
        ims = [self._reshapeTorch(renderVectors[i]) for i in range(N)]
        return torch.stack(ims)

    def renderingNP(self, ims):
        matT = self.matT
        nPixels = matT.shape[0]
        N, H, W, C = ims.shape
        assert H * W == matT.shape[1], '[ERROR] transportMat %d doesn\'t match image size %d' % (H * W, matT.shape[1])

        # operation from matlab, reshape in column order
        ims = np.transpose(ims, axes=[0, 2, 1, 3])
        lights = np.reshape(ims, (-1, H * W, C))  # [N, #pixel, C]
        # transpose and reshape
        lightsT = np.transpose(lights, axes=[1, 0, 2])  # [W*H, N, C]
        lightsTR = np.reshape(lightsT, [W * H, -1])  # [W*H, N*C]
        vectors = np.matmul(matT, lightsTR)  # [#pixel, W*H] x [W*H, N*C)] -> [#pixel, N*C]
        vectors = np.reshape(vectors, [nPixels, -1, C])  # [#pixel, N, C]
        vectors = np.transpose(vectors, axes=[1, 0, 2])  # [N, #pixel, C]
        renderVectors = vectors
        return renderVectors

    def _reshapeNP(self, tl_WHxC):
        # todo batch reshape
        C = tl_WHxC.shape[1]
        img = np.zeros((self.maskT.shape[0], self.maskT.shape[1], C))
        img[self.maskT] = tl_WHxC
        img = np.ndarray.transpose(img, [1, 0, 2])
        return img

    def reshapeNP(self, renderVectors):
        N = renderVectors.shape[0]
        ims = [self._reshapeNP(renderVectors[i]) for i in range(N)]
        return np.asarray(ims)

    def render_top_down(self, img,chw=True):
        if chw:
            img = img.permute(0,2,3,1)
        if type(img).__module__.find('numpy') >= 0:
            reverse_img = np.flip(img, [1])

        elif type(img).__module__.find('torch') >= 0:
            reverse_img = torch.flip(img, [1])
        else :
            print('[ERROR] Input img is neither torch nor numpy')

        return {"top" : self.rendering(img, dst='image'), "bottom": self.rendering(reverse_img, dst='image')}

def example():
    import time
    im_h = 256
    img_dim = 64
    rtm = RenderWithTransportMat(transportMatFname='transportMat.BumpySphereMiddle.top.e64.r64.half.mat', lightHeight=img_dim, doHalfMat=True)
    ims = np.ones([256, img_dim, img_dim*2, 3])
    print(ims.shape)
    # ims shape = [b,h,w,c]
    # pred shape = [b,c,h,w]

    # pred = [256,512]
    # pred = [64,128] = > [32,128]*2
    # what it wants = [128,64]=> [64,64]*2
    p = rtm.render_top_down(torch.from_numpy(ims).type('torch.FloatTensor'))
    print(p['top'].shape)
    print(p['bottom'].shape)

if __name__ == '__main__':
    import sys
    print(sys.version)
    example()

{'lz4.frame': 'lz4.frame', 'msgpack': 'msgpack', 'msgpack_numpy': 'msgpack_numpy', 'partial': 'functools.partial'}


from flask import Blueprint, render_template, redirect, url_for
from project import db
from project.models import Ponny
from project.ponnies.forms import AddForm, DelForm

ponnies_blueprint = Blueprint(
    'ponnies', __name__, template_folder='templates/ponnies')

@ponnies_blueprint.route('/add', methods=['GET', 'POST'])
def add():
    form = AddForm()

    if form.validate_on_submit():
        name = form.name.data
        new_pon = Ponny(name)
        db.session.add(new_pon)
        db.session.commit()
        return redirect(url_for('ponnies_list'))
    return render_template('add.html', form=form)

@ponnies_blueprint.route('/liste')
def liste():
    ponnies = Ponny.query.all()
    return render_template('list.html', ponnies=ponnies)

@ponnies_blueprint.route('/delete', methods=['GET', 'POST'])
def delete():
    form = DelForm()
    if form.validate_on_submit():
        id = form.id.data
        pon = Ponny.query.get(id)
        db.session.delete(pon)
        db.session.commit()
        return redirect(url_for('ponnies.liste'))
    return render_template('delete.html', form=form)


import geopandas as gpd
import re
from shapely import wkt
import folium
import streamlit as st
import streamlit_folium

try:
    uploaded_file = st.file_uploader("Please Upload GeoJSON or ZIP Shapefiles here")
    geofence_gdf = gpd.read_file(uploaded_file)
    geofence_gdf['str_geom'] = geofence_gdf.geometry.apply(lambda x: wkt.dumps(x))

    # CONVERT QGIS GEOMETRY TO GEOTOOLS COORDINATES
    for i, row in geofence_gdf.iterrows():
       # EXTRACTING THE COORDINATES FROM STR GEOM USING REGEX
       coordinates_match = re.search(r'POLYGON \(\((.*?)\)\)', row['str_geom'])
    if coordinates_match:
        coordinates_text = coordinates_match.group(1)
        coordinates = coordinates_text.split(', ')

          #REFORMATING THE COORDINATES ACCORDING GEOTOOLS
        reformatted_coordinates = []
        for coord in coordinates:
            x,y = coord.split(' ')
            reformatted_coordinates.append(f'{y}, {x}')

        #JOINING THE REFORMATTED COORDINATES WITH COMMAS
            result_string = ', '.join(reformatted_coordinates)
            geofence_gdf.at[i, 'geotools_coordinates'] = result_string
        else:
            st.write("invalid input string")

    output = st.text_input('File Save As', )
    csv=geofence_gdf.to_csv()
    st.download_button(
      label="Download data as CSV",
      data=csv,
      file_name = output+'.csv',
      mime='text/csv',
  )

except (TypeError, NameError, AttributeError):
  pass