#!/usr/bin/env python

"""
CMSC733 Spring 2019: Classical and Deep Learning Approaches for
Geometric Computer Vision
Homework 0: Alohomora: Phase 2 Starter Code

Author(s):
Nitin J. Sanket (nitinsan@terpmail.umd.edu)
PhD Candidate in Computer Science,
University of Maryland, College Park
"""

import tensorflow as tf
import cv2
import os
import sys
import glob
import Misc.ImageUtils as iu
import random
from skimage import data, exposure, img_as_float
import matplotlib.pyplot as plt
from Network.Network import HomographyModel
from Misc.MiscUtils import *
import numpy as np
import time
import argparse
import shutil
from StringIO import StringIO
import string
import math as m
from tqdm import tqdm
from Misc.TFSpatialTransformer import *

# Don't generate pyc codes
sys.dont_write_bytecode = True

def SetupAll(BasePath):
    """
    Inputs:
    BasePath - Path to images
    Outputs:
    ImageSize - Size of the Image
    DataPath - Paths of all images where testing will be run on
    """
    # Image Input Shape
    ImageSize = [64, 64, 2]
    DataPath = []
    NumImages = len(glob.glob(BasePath+'*.jpg'))
    SkipFactor = 1
    for count in range(1, NumImages+1, SkipFactor):
        DataPath.append(BasePath + str(count) + '.jpg')

    return ImageSize, DataPath

def ReadImages(ImageSize, DataPath):
    """
    Inputs:
    ImageSize - Size of the Image
    DataPath - Paths of all images where testing will be run on
    Outputs:
    I1Combined - I1 image after any standardization and/or cropping/resizing to ImageSize
    I1 - Original I1 image for visualization purposes only
    """

    ImageName = DataPath
    Image = cv2.imread(ImageName)
    Gray = cv2.cvtColor(Image, cv2.COLOR_BGR2GRAY)

    if (Image is None):
        # OpenCV returns empty list if image is not read!
        print('ERROR: Image cannot be read')
        sys.exit()

    Tau = min(ImageSize[0], ImageSize[1])/3
    # generate label
    Label = np.random.randint(2*Tau, size=8) - Tau

    # get corner correspondences
    Corners, Warped = iu.getCorrespondence(Image.shape, Label, ImageSize)

    # get forward and backward homographies
    CroppedPatch, WarpedPatch = iu.getWarpingPair(Gray, ImageSize, Corners, Warped)
    I1 = np.float32(np.stack([CroppedPatch, WarpedPatch], -1))

    #cv2.imshow("Image", Image)
    #cv2.imshow("Warped Patch", WarpedPatch)
    #cv2.imshow("Cropped Patch", CroppedPatch)

    I1S = iu.StandardizeInputs(np.float32(I1))

    I1Combined = np.expand_dims(I1S, axis=0)

    return I1Combined, Image, Label, Corners

def BoundingBox(Img, Corners, TrueLabel, PredLabel, count):

    TrueWarped = np.array([(Corners[0, 0] + TrueLabel[0], Corners[0, 1] + TrueLabel[1]),
                           (Corners[1, 0] + TrueLabel[2], Corners[1, 1] + TrueLabel[3]),
                           (Corners[2, 0] + TrueLabel[4], Corners[2, 1] + TrueLabel[5]),
                           (Corners[3, 0] + TrueLabel[6], Corners[3, 1] + TrueLabel[7])], dtype=np.float32)

    PredWarped = np.array([(Corners[0, 0] + PredLabel[0], Corners[0, 1] + PredLabel[1]),
                           (Corners[1, 0] + PredLabel[2], Corners[1, 1] + PredLabel[3]),
                           (Corners[2, 0] + PredLabel[4], Corners[2, 1] + PredLabel[5]),
                           (Corners[3, 0] + PredLabel[6], Corners[3, 1] + PredLabel[7])], dtype=np.float32)

    for i in range(4):
        cv2.circle(Img, (Corners[i, 0], Corners[i, 1]), 2, (255,0,0), 3)
        cv2.circle(Img, (TrueWarped[i, 0], TrueWarped[i, 1]), 2, (0,0,255), 3)
        cv2.circle(Img, (PredWarped[i, 0], PredWarped[i, 1]), 2, (0,255,255), 3)
    cv2.polylines(Img, np.int32([Corners]), True, (255,0,0))
    cv2.polylines(Img, np.int32([TrueWarped]), True, (0,0,255))
    cv2.polylines(Img, np.int32([PredWarped]), True, (0,255,255))
    cv2.imwrite("ykwang11_p1/Phase2/Output/%d.png" %count, Img)
    #cv2.imshow("result", Img)
    #cv2.waitKey()

def TestOperation(ImgPH, ImageSize, ModelPath, DataPath):
    """
    Inputs:
    ImgPH is the Input Image placeholder
    ImageSize is the size of the image
    ModelPath - Path to load trained model from
    DataPath - Paths of all images where testing will be run on
    LabelsPathPred - Path to save predictions
    Outputs:
    Predictions written to ./TxtFiles/PredOut.txt
    """
    Length = ImageSize[0]
    # Predict output with forward pass, MiniBatchSize for Test is 1
    H4Pt, _ = HomographyModel(ImgPH, ImageSize, 1)

    # Setup Saver
    Saver = tf.train.Saver()

    errorSum = 0
    with tf.Session() as sess:
        Saver.restore(sess, ModelPath)
        print('Number of parameters in this model are %d ' % np.sum([np.prod(v.get_shape().as_list()) for v in tf.trainable_variables()]))

        for count in tqdm(range(np.size(DataPath))):
            DataPathNow = DataPath[count]
            Img, ImgOrg, Label, Corners = ReadImages(ImageSize, DataPathNow)

            FeedDict = {ImgPH: Img}
            Pred = sess.run(H4Pt, FeedDict)
            #print ("pred ", Pred)
            #print ("label ", Label)
            #print ("---------> ", Corners)
            errorSum += np.aqrt(np.mean((Pred - Label)*(Pred - Label)))
            #print ("---------> ", Pred)
            BoundingBox(ImgOrg, Corners, Label, Pred.reshape(-1), count)

    print('The average 2-norm error is %f ' % (errorSum / np.size(DataPath)))

def main():
    """
    Inputs:
    None
    Outputs:
    Prints out the confusion matrix with accuracy
    """

    # Parse Command Line arguments
    Parser = argparse.ArgumentParser()
    Parser.add_argument('--ModelType', default='Unsup', help='Model type, Supervised or Unsupervised? Choose from Sup and Unsup, Default:Unsup')
    Parser.add_argument('--CheckPointPath', default='ykwang11_p1/Phase2/Checkpoints/', help='Path to save Checkpoints, Default: ykwang11_p1/Phase2/Checkpoints/')
    Parser.add_argument('--BasePath', dest='BasePath', default='ykwang11_p1/Phase2/Data/Val/', help='Path to load images from, Default:BasePath')
    Args = Parser.parse_args()
    ModelType = Args.ModelType
    CheckPointPath = Args.CheckPointPath + ModelType + '/'
    BasePath = Args.BasePath

    # Load lastest checkpoint
    LatestFile = FindLatestModel(CheckPointPath)
    ModelPath =  CheckPointPath + LatestFile + '.ckpt'

    # Setup all needed parameters including file reading
    ImageSize, DataPath = SetupAll(BasePath)

    # Define PlaceHolder variables for Input and Predicted output
    ImgPH = tf.placeholder('float', shape=(1, ImageSize[0], ImageSize[1], ImageSize[2]))

    print(ModelPath)
    print(DataPath)
    TestOperation(ImgPH, ImageSize, ModelPath, DataPath)

if __name__ == '__main__':
    main()


from datetime import datetime

class Greeter:
  def __init__(self, name, store) -> None:
    self.name = name
    self.store = store

  def _day(self):
    return datetime.now().strftime('%A')

  def _part_of_day(self):
    hour = datetime.now().hour
    if (hour < 12):
      return 'Morning'
    elif (hour < 17):
      return 'Afternoon'
    else:
      return 'Evening'

  def greet(self):
    print(f'Hi, I am {self.name}, and welcome to {self.store}')
    print(f'How\'s your {self._day()} {self._part_of_day()} going?')
    print('Hope you are doing well!')

greeter = Greeter('Yen', 'Crab Soup')
greeter.greet()


#!/usr/bin/env python3

""" The basic plan:

1. Due to memory constraints, train up a model one day at a time.
2. Load up one day's data. Use online training with xgboost to update the model.
3. Validate using the final month's worth of data.

"""

import glob
import re

import matplotlib
matplotlib.use("agg")
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np

import xgboost as xgb

# I am trying something weird with loss functions here as I am not sure
# how to do this! The idea is that I replace the +/- with a 1/0 loss function
# and then weight according to the weightings.
def load_day(day_number, filename_ds):
    """ Load up a CSV for a given day.

    Days run from 0.

    """

    example_filename = filename_ds.loc[filename_ds.date_number==day_number].filename.to_numpy()[0]
    this_ds = pd.read_csv(example_filename).sort_values(["date", "ts_id"])

    this_ds["good_trade"] = ((this_ds.resp * this_ds.weight) > 0.0).astype(np.int)
    this_ds["trade_weight"] = (this_ds.resp * this_ds.weight).abs()
    this_ds["trade_value"] = this_ds.resp * this_ds.weight

    return this_ds

# First up - data load.
# The files have been split into one day / file chunks.
day_files = [day_file for day_file in glob.glob("data/splitfiles/day*.csv")]
day_file_numbers = [(int(re.search("\d+", day_file).group(0)), day_file)
                    for day_file in day_files]
day_file_numbers.sort(key=lambda x: x[0])
day_file_numbers = pd.DataFrame(day_file_numbers, columns=["date_number", "filename"])

# Load the features and example testing data
features = pd.read_csv("data/features.csv")
example_test = pd.read_csv("data/example_test.csv")

# We'll do a 80% train/validation split
train_split = 400
day_file_numbers["train"] = day_file_numbers.date_number < train_split

# Do some data analysis, because this data is all pretty weird.
first_day = load_day(0, day_file_numbers)

# Look at histograms of weights and responses
_, ax = plt.subplots(figsize=(16, 9))
ax.hist(first_day.weight)
plt.savefig("weight_histogram.png")

_, ax = plt.subplots(figsize=(16, 9))
ax.hist(first_day.resp)
plt.savefig("response_histogram.png")

# Plan: train a model for the first day. Then, for every other day we just nudge it.
num_trees = 50
params = {"booster": "gbtree",
          "eta": 0.5,
          "gamma": 2.0,
          "max_depth": 6,
          "lambda": 1.1,
          "colsample_bytree": 0.75,
          "process_type": "default",
          "objective": "reg:squarederror"}

# Now, we begin a training loop. One day at a time.
# There are 130 feature columns
num_rounds = 250
early_stopping = 10
feature_cols = ["feature_{}".format(i) for i in range(130)]

# Maybe I want this multiple times?
first_day = True
for day_number in day_file_numbers.loc[day_file_numbers.train].date_number:
    print("processing day: {}".format(day_number+1))

    # Once we move to day 1, update rather than build new model.
    if not first_day:
        params["process_type"] = "update"
        params["updater"] = "refresh"
        params["refresh_leaf"] = 1
        params["eta"] = 0.1

    # Grab a random day to test against (this is going to get weird)
    test_day = day_file_numbers.loc[~day_file_numbers.train].date_number.sample()

    train_data = load_day(day_number, day_file_numbers)
    test_data = load_day(test_day.to_numpy()[0], day_file_numbers)

    dtrain = xgb.DMatrix(train_data[feature_cols],
                         label=train_data.trade_value)

    dvalid = xgb.DMatrix(test_data[feature_cols],
                         label=test_data.trade_value)

    evallist = [(dvalid, "eval"), (dtrain, "train")]

    if first_day:
        # TB NOTE: Don't do early stopping, I want all the trees.
        bst = xgb.train(params, dtrain, num_rounds, evallist)
    else:
        bst= xgb.train(params, dtrain, num_rounds, evallist,
                       xgb_model=bst, early_stopping_rounds=early_stopping)

    first_day = False

# Predict on the output
dtest = xgb.DMatrix(example_test[feature_cols])
example_test["results"] = bst.predict(dtest)

example_test["output"] = example_test.results > 0.0


import bs4 as bs
import requests
import time
import datetime as dt
import pickle
from secret_information import STOCK_API_KEY

ERROR_VALUE = 0.0001 #This is the value that gets added to our data if we cannot find what we are looking for.

wiki_response = requests.get('http://en.wikipedia.org/wiki/List_of_S%26P_500_companies')
soup = bs.BeautifulSoup(wiki_response.text, 'lxml')
table = soup.find('table', {'class': 'wikitable sortable'})

tickers = []
# Tickers is a variable that stores all of the stock tickers

# The code below finds the snp500 stock table on wikipedia and takes all of the stocks and adds them to tickers
# Stratosphere only accepts BF-B instead of BF.B so we have to change the stock's name manually.
for row in table.findAll('tr')[1:]:  # We skip the tr header
    ticker = row.findAll('td')[0].text
    ticker = ticker.strip()
    if ticker == "BF.B":
        ticker = "BF-B"
    elif ticker == "BRK.B":
        ticker = "BRK-B"
    tickers.append(ticker)

def get_number(string):
    """ When we receive a response from stratosphere, this isolates the number part of the
    information and returns it as a float """
    number = string.split(",")[0].split("}")[0].split(":")[1]
    return float(number)

def calculate_score(value, good_number, bad_number):
    """ This scores a particular stock's criteria, ideally we want it to be below the good number,
    however if the stock is a little bit over then the score is only increased a little bit"""
    output = 0
    if value == ERROR_VALUE:
        output += 1000
    elif value <= good_number:
        pass
    elif value < bad_number:
        percentage_off = (value - good_number) * 100
        output += percentage_off
    else:
        output += 1000
    return output

parameters = {
    "function": "overview",
    "symbol": "This is the stock we are looking for - this will change when we run the code",
    "apikey": STOCK_API_KEY,
}
# These parameters will be used when we send a request to alphavantage's API

our_data = {}
good_stocks = {}
counter = 0
# Alpha vantage only lets us send 5 requests per minute, the count variable helps us limit our requests

# For some reason KO broke the code - the stock doesnt meet our criteria anyways so
# We do not need to check it.
broken_stocks = ["KO"]
for broken_stock in broken_stocks:
    tickers.remove(broken_stock)
stocks_done = 0

for stock in tickers:
    parameters["symbol"] = stock
    stocks_done += 1
    if stocks_done > 500:
        break
    if counter == 5:
        time.sleep(11)
        # The sleep statements workout to 61 seconds of sleep between batches of 5
        # This means we are not spamming the api.
        counter = 0
    counter += 1
    print(f"Now working on {stock}")
    try:
        response = requests.get(url="https://www.alphavantage.co/query", params=parameters)
        data = response.json()
    except:
        data["PERatio"] = ERROR_VALUE
        data["Beta"] = ERROR_VALUE

    try:
        pe_ratio = float(data["PERatio"])
    except:
        pe_ratio = ERROR_VALUE
    try:
        beta = float(data["Beta"])
    except:
        beta = ERROR_VALUE
    try:
        response = requests.get("https://www.stratosphere.io/company-search/" + stock + "/")
        # We extract the data that we need from stratosphere
    except:
        response = f"totaldebtequity:{ERROR_VALUE}, pricetosales:{ERROR_VALUE}"
    time.sleep(10)  # Incase they get angry at us spamming their website - 5* 1 + 56 > 60
    try:
        debt_to_equity = response.text.split("totaldebttoequity")[1]
        debt_to_equity = get_number(debt_to_equity)
    except:
        debt_to_equity = ERROR_VALUE
    try:

        price_to_sales = response.text.split("pricetosales")[1]
        price_to_sales = get_number(price_to_sales)
    except:
        price_to_sales = ERROR_VALUE

    score = 0

    score += calculate_score(debt_to_equity, 1, 2)
    score += calculate_score(price_to_sales, 3, 5)
    score += calculate_score(pe_ratio, 25, 35)
    score += calculate_score(beta, 1, 10)

    if score <= 10:
        good_stocks[stock] = score

    mini_json = {}
    mini_json["pe ratio"] = pe_ratio
    mini_json["beta"] = beta
    mini_json["debt to equity"] = debt_to_equity
    mini_json["price to sales"] = price_to_sales
    mini_json["score"] = score
    print(mini_json)
    our_data[stock] = mini_json

# The variable our_data is a dictionary containing the data that we want for each stock
# We will save this data for later use

date = dt.datetime.today()
with open(f"Snp500_stock_dict_{date.month}_{date.year}", "wb") as f:
    pickle.dump(our_data, f)

print(f"The value stocks are: {good_stocks}")

# If we want to save our data to a csv:
data = our_data
df = data.stack().reset_index().rename(index=str, columns={"level_1": "Symbol"}).sort_values(['Symbol', 'Date'])
df = df.set_index('Date', inplace=True)

df.to_csv(f"SnP_500_Stocks{date.month}_{date.year}.csv")


import ast
from dataclasses import dataclass
from pathlib import Path

from sphinx.application import Sphinx
from sphinx.util.logging import getLogger

logger = getLogger(__name__)

@dataclass
class Example:
    path: Path
    source_folder: Path
    name: str = ""
    reference: str = ""
    summary: str = ""
    category: str = ""

    def __post_init__(self) -> None:
        self.name = self.path.name
        has_subdir = len(self.path.relative_to(self.source_folder.parent).parts) > 2
        if has_subdir:
            self.reference = f"{self.path.parts[-2]}-{self.path.stem}"
            self.category = self.path.parts[-2]
        else:
            self.reference = self.path.stem

        self.reference = self.reference.replace("_", "-").replace(" ", "")

@dataclass
class PythonExample(Example):
    def __post_init__(self) -> None:
        super().__post_init__()

        mod = ast.parse(self.path.read_bytes())
        doc = ""
        for node in mod.body:
            if isinstance(node, ast.Expr) and isinstance(node.value, ast.Str):
                doc = node.value.s.strip().split("\n\n")[0].strip()
                if not doc.endswith("."):
                    doc += "."
                break
        self.summary = doc

def md_generator(extension: str, app: Sphinx, source_folder: Path) -> list[Example]:
    examples: list[Example] = []
    for ff in source_folder.rglob(extension):
        pth = ff.relative_to(app.srcdir)
        if " " in str(pth):
            logger.warning(
                f"The example '{pth!s}' has a space in the "
                "pathname which is not yet supported."
            )
            continue
        if ff.suffix == ".py":
            example = PythonExample(ff, source_folder)
        else:
            example = Example(ff, source_folder)  # type: ignore
        examples.append(example)
        # md_file = ff.with_suffix(".md")
        # md_file.write_text(
        #     dedent(
        #         f"""\
        #         ---
        #         orphan: true
        #         ---
        #         ({example.reference})=
        #         # {example.name}

        #         [**Source**]({ff.name})

        #         :::{{literalinclude}} {ff.name}
        #         :::
        #         """
        #     )
        # )
    return examples


import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split

class Preprocessing:
    df = pd.read_csv('ce889_dataCollection.csv',header = None)
    # for dropping the duplicates
    df.drop_duplicates(inplace = True)
    # checking for null values
    print("Checking For Null values", df.isnull().sum())
    # Giving column names
    df.columns = ["X distance to target","Y distance to target","New Vel Y","New Vel X"]
    # normalization
    data_min = df.min()
    data_max = df.max()
    normalizeddata = (df - df.min()) / (df.max() - df.min())
    # spliting data
    training , testing= train_test_split(normalizeddata, test_size = 0.2)
    print("Shape of Training Data", training.shape)
    print("Shape of Testing Data", testing.shape)
    # Splitting inputs & outputs for training, validation & testing data
    x_train_input = np.array(training[["X distance to target","Y distance to target"]]).T
    y_train_output = np.array(training[["New Vel Y","New Vel X"]]).T

    x_testing_input = np.array(testing[["X distance to target","Y distance to target"]]).T
    y_testing_output = np.array(testing[["New Vel Y","New Vel X"]]).T

Rocket Landing Neural Netwotk/Neural Network Python Code/Data_Preprocessing.py


"""
基本种类:
:: EVENT [WIDGET]      | 一般在文件开头

<<WIDGET_1>>ANY<</WIDGET_1>>   | 有闭合的
<<WIDGET_0>>                | 无闭合的
<<WIDGET "TAG">>            | 特殊名称
<<WIDGET COND>>             | 条件，如 if / link 等

特殊:
<<if>>ANY<<elseif>>ANY<<else>>ANY<</if>>

<<switch>>ANY<<case>>ANY<</switch>>

<<link [[TEXT]]>></link<>>
<<link [[TEXT|EVENT]]>></link<>>

<<set $VAR to KEYWORD>>
<<set $VAR to "VAR">>
<<set $VAR to "STRING">>

$VAR                | 变量
$VAR.FUNC(PARAM)    | 调用函数
$VAR.PROP           | 属性

STRING                  | 文本
<CHAR_1>ANY</CHAR_1>    | 有闭合的
<CHAR_0>                | 无闭合的

TEXT

//COMMENT       | 注释
/*COMMENT*/     | 注释(可跨行)
    /* COMMENT
     * COMMENT
     * COMMENT */
  | 注释(可跨行)

要翻译的：
TEXT, STRING
"""
import asyncio
import time

from src import (
    logger,
    Paratranz,
    ProjectDOL,
    PARATRANZ_TOKEN
)

async def main():
    start = time.time()
    # =====
    dol = ProjectDOL(type_="common")  # 改成 “dev” 则下载最新开发版分支的内容
    pt = Paratranz()
    if not PARATRANZ_TOKEN:
        logger.error("未填写 PARATRANZ_TOKEN, 汉化包下载可能失败，请前往 https://paratranz.cn/users/my 的设置栏中查看自己的 token, 并在 src/consts.py 中填写\n")
        return

    """ 删库跑路 """
    await dol.drop_all_dirs()

    """ 获取最新版本 """
    await dol.fetch_latest_version()

    """ 提取键值 """
    await dol.download_from_gitgud()
    await dol.create_dicts()

    """ 更新导出的字典 成品在 `raw_dicts` 文件夹里 """
    download_flag = await pt.download_from_paratranz()  # 如果下载，需要在 consts 里填上管理员的 token, 在网站个人设置里找
    if not download_flag:
        return
    await dol.update_dicts()

    """ 覆写汉化 用的是 `paratranz` 文件夹里的内容覆写 """
    await dol.apply_dicts()

    """ 编译成游戏 """
    dol.compile()
    dol.run()
    # =====
    end = time.time()
    return end-start

if __name__ == '__main__':
    last = asyncio.run(main())
    logger.info(f"===== 总耗时 {last or -1}s =====")


"""Copy all orginal screenings to a new directory."""

import re
import glob
import shutil
import numpy as np
from PIL import Image
from utils import colorize

import os
from joblib import Parallel, delayed
from settings import (
    ANONYMIZED_DATA_DIRECTORY, IMAGES_DIRECTORY)

def copy_image(origin_folder, target_folder):
    """
    Find original screening in origin_folder and copy it to target_folder.
    """

    # Define path to all png files in the folder
    path = origin_folder + "/*.png"

    # Create list with addresses of all files in the folder
    addrs = sorted(glob.glob(path))

    # Get screenings path
    all_images = re.compile(".*_[0-9].png")
    image_list = list(filter(all_images.match, addrs))
    number_of_images = len(image_list)

    # If no images
    if number_of_images == 0:
        all_images = re.compile(".*_[0-9][0-9].png")
        image_list = list(filter(all_images.match, addrs))

    # Copy to new folder
    for image in image_list:

        # Check if image is RGB or 8bit
        img = np.array(Image.open(image))

        if len(img.shape) != 2:
            print(img.shape)
            print(colorize("{}".format(origin_folder), "red"))

        shutil.copy(image, target_folder)

if __name__ == "__main__":

    if not os.path.isdir(IMAGES_DIRECTORY):
        os.mkdir(IMAGES_DIRECTORY)
    list_of_patient_dirs = glob.glob(os.path.join(ANONYMIZED_DATA_DIRECTORY, "*"))
    Parallel(n_jobs=-1, verbose=1)(
        delayed(copy_image)(
            origin_folder=patient_dir,
            target_folder=IMAGES_DIRECTORY,
            )
        for patient_dir in sorted(list_of_patient_dirs))


# Import numpy
import numpy as np

# Import matplotlib
import matplotlib.pylab as plt

# Configure
plt.rcParams['xtick.direction'] = 'out'
plt.rcParams['ytick.direction'] = 'out'

# Optionally set font to Computer Modern to avoid common missing font errors
params = {
  'axes.labelsize': 24,
  'legend.fontsize': 14,
  'xtick.labelsize': 24,
  'ytick.labelsize': 24,
  'text.usetex': True}
plt.rcParams.update(params)

# Latex math
plt.rcParams['text.latex.preamble'] = [r'\usepackage{sfmath}']
plt.rcParams['font.family'] = 'sans-serif'
# plt.rcParams['font.sans-serif'] = 'courier'
plt.rcParams['font.size'] = 18
plt.rcParams['font.weight'] = 'bold'
plt.rcParams['lines.linewidth'] = 4
plt.rcParams['lines.color'] = 'r'

# Make sure everything is within the frame
plt.rcParams.update({'figure.autolayout': True})

# Set marker size
markerSize = 11.0

# bar chart settings
alpha = 0.8

# These are the "Tableau 20" colors as RGB.
tableau20 = [(31, 119, 180), (174, 199, 232), (255, 127, 14), (255, 187, 120),
             (44, 160, 44), (152, 223, 138), (214, 39, 40), (255, 152, 150),
             (148, 103, 189), (197, 176, 213), (140, 86, 75), (196, 156, 148),
             (227, 119, 194), (247, 182, 210), (127, 127, 127), (199, 199, 199),
             (188, 189, 34), (219, 219, 141), (23, 190, 207), (158, 218, 229)]

# Scale the RGB values to the [0, 1] range, which is the format matplotlib accepts.
for i in range(len(tableau20)):
    r, g, b = tableau20[i]
    tableau20[i] = (r / 255., g / 255., b / 255.)

mevery = 3

class PostOpt:
    """
    Class to handle plotting
    """
    @staticmethod
    def plot_solution(data1, data2, name):
        plt.figure()
        fig, ax = plt.subplots()

        # remove the upper and right borders
        ax.spines['right'].set_visible(False)
        ax.spines['top'].set_visible(False)

        ax.xaxis.set_ticks_position('bottom')
        ax.yaxis.set_ticks_position('left')

        ax.plot(data1[:,0], data1[:,1], '-D', label='(a) cg',
                     ms=markerSize, mec='black', color=tableau20[0], markevery=mevery,
                     alpha=alpha)
        ax.plot(data1[:,0], data1[:,2], '-v', label='(a) pcg',
                     ms=markerSize, mec='black', color=tableau20[0], markevery=mevery,
                     alpha=alpha)
        ax.plot(data1[:,0], data1[:,3], '-*', label='(a) exact',
                     ms=markerSize, mec='black', color=tableau20[0], markevery=mevery,
                     alpha=alpha)

        ax.plot(data2[:,0], data2[:,1], '-D', label='(b) cg',
                     ms=markerSize, mec='black', color=tableau20[2], markevery=mevery,
                     alpha=alpha)
        ax.plot(data2[:,0], data2[:,2], '-v', label='(b) pcg',
                     ms=markerSize, mec='black', color=tableau20[2], markevery=mevery,
                     alpha=alpha)
        ax.plot(data2[:,0], data2[:,3], '-*', label='(b) exact',
                     ms=markerSize, mec='black', color=tableau20[2], markevery=mevery,
                     alpha=alpha)

        ax.set_ylabel('Solution u(x)')
        ax.set_xlabel('Domain x')
        #niters_labl = [1, 20, 40, 60, 80, 100]
        #ax.set_yticks(np.logspace(-5, 0, 6, endpoint=True))
        #ax.set_yticks(np.linspace(0, 1, 5, endpoint=True, dtype=np.int))
        ax.set_yticks(np.linspace(0, 1.5, 4, endpoint=True))
        ax.legend(loc='lower left', framealpha=0.0)

        plt.savefig(name, bbox_inches='tight', pad_inches=0.05)

        return

if __name__ == "__main__":

    inpFile = open("dirichlet.dat", "r")
    fdirichlet = list(inpFile.readlines())
    inpFile.close()

    inpFile = open("mixed.dat", "r")
    fmixed = list(inpFile.readlines())
    inpFile.close()

    soldirichlet = []
    for line in fdirichlet:
        entry = line.split()
        soldirichlet.append([float(entry[0]),
                             float(entry[1]),
                             float(entry[2]),
                             float(entry[3])])
    soldirichlet = np.array(soldirichlet)

    solmixed = []
    for line in fmixed:
        entry = line.split()
        solmixed.append([float(entry[0]), float(entry[1]), float(entry[2]), float(entry[3])])
    solmixed = np.array(solmixed)

    # Plot the solution in the profile
    PostOpt.plot_solution(soldirichlet, solmixed, "profile.pdf")


import os
from pathlib import Path

from .utils import CREDENTIALS_FILE
from .path_resolver import resolve_cwd

def get_credentials(root_path: Path):
    if os.getenv("ABSTRA_API_TOKEN"):
        return os.getenv("ABSTRA_API_TOKEN")

    credentials_path = Path(root_path, CREDENTIALS_FILE)
    if not credentials_path.exists():
        return None

    return credentials_path.read_text(encoding="utf-8").strip()

def delete_credentials(root_path: Path):
    credentials_path = Path(root_path, CREDENTIALS_FILE)

    if credentials_path.exists():
        credentials_path.unlink()

def set_credentials(root_path: Path, token: str):
    credentials_path = Path(root_path, CREDENTIALS_FILE)
    credentials_path.parent.mkdir(exist_ok=True)

    credentials_path.write_text(token, encoding="utf-8")

def resolve_headers(root: Path = Path(".")):
    root_path = resolve_cwd(root)
    credentials = get_credentials(root_path)

    if not credentials:
        return None

    return {"Api-Authorization": f"Bearer {credentials}"}


from typing import List

class Solution:
    def findMaximizedCapital(self, k: int, w: int, profits: List[int], capital: List[int]) -> int:
        """
        This will TLE
        32 / 35 testcases passed
        """
        project_used = set()
        while k:
            temp_p = 0
            temp_i = -1
            for i, (p, c) in enumerate(zip(profits, capital)):
                if i in project_used:
                    continue
                if c <= w and p > temp_p:
                    temp_p = p
                    temp_i = i
            project_used.add(temp_i)
            w += temp_p
            k -= 1

        return w


import dbus
import dbus.service

import gpodder
from gpodder import util

def safe_str(txt):
    if txt:
        return txt.encode()
    else:
        return ''

def safe_first_line(txt):
    txt = safe_str(txt)
    lines = util.remove_html_tags(txt).strip().splitlines()
    if not lines or lines[0] == '':
        return ''
    else:
        return lines[0]

class DBusPodcastsProxy(dbus.service.Object):
    """ Implements API accessible through D-Bus

    Methods on DBusPodcastsProxy can be called by D-Bus clients. They implement
    safe-guards to work safely over D-Bus while having type signatures applied
    for parameter and return values.
    """

    # DBusPodcastsProxy(lambda: self.channels, self.on_itemUpdate_activate(), self.playback_episodes, self.download_episode_list, bus_name)
    def __init__(self, get_podcast_list,
            check_for_updates, playback_episodes,
            download_episodes, episode_from_uri,
            bus_name):
        self._get_podcasts = get_podcast_list
        self._on_check_for_updates = check_for_updates
        self._playback_episodes = playback_episodes
        self._download_episodes = download_episodes
        self._episode_from_uri = episode_from_uri
        dbus.service.Object.__init__(self,
                object_path=gpodder.dbus_podcasts_object_path,
                bus_name=bus_name)

    def _get_episode_refs(self, urls):
        """Get Episode instances associated with URLs"""
        episodes = []
        for p in self._get_podcasts():
            for e in p.get_all_episodes():
                if e.url in urls:
                    episodes.append(e)
        return episodes

    @dbus.service.method(dbus_interface=gpodder.dbus_podcasts, in_signature='', out_signature='a(ssss)')
    def get_podcasts(self):
        """Get all podcasts in gPodder's subscription list"""
        def podcast_to_tuple(podcast):
            title = safe_str(podcast.title)
            url = safe_str(podcast.url)
            description = safe_first_line(podcast.description)
            cover_file = ''

            return (title, url, description, cover_file)

        return [podcast_to_tuple(p) for p in self._get_podcasts()]

    @dbus.service.method(dbus_interface=gpodder.dbus_podcasts, in_signature='s', out_signature='ss')
    def get_episode_title(self, url):
        episode = self._episode_from_uri(url)

        if episode is not None:
            return episode.title, episode.channel.title

        return ('', '')

    @dbus.service.method(dbus_interface=gpodder.dbus_podcasts, in_signature='s', out_signature='a(sssssbbb)')
    def get_episodes(self, url):
        """Return all episodes of the podcast with the given URL"""
        podcast = None
        for channel in self._get_podcasts():
            if channel.url == url:
                podcast = channel
                break

        if podcast is None:
            return []

        def episode_to_tuple(episode):
            title = safe_str(episode.title)
            url = safe_str(episode.url)
            description = safe_first_line(episode._text_description)
            filename = safe_str(episode.download_filename)
            file_type = safe_str(episode.file_type())
            is_new = (episode.state == gpodder.STATE_NORMAL and episode.is_new)
            is_downloaded = episode.was_downloaded(and_exists=True)
            is_deleted = (episode.state == gpodder.STATE_DELETED)

            return (title, url, description, filename, file_type, is_new, is_downloaded, is_deleted)

        return [episode_to_tuple(e) for e in podcast.get_all_episodes()]

    @dbus.service.method(dbus_interface=gpodder.dbus_podcasts, in_signature='as', out_signature='(bs)')
    def play_or_download_episode(self, urls):
        """Play (or download) a list of episodes given by URL"""
        episodes = self._get_episode_refs(urls)
        if not episodes:
            return (0, 'No episodes found')

        to_playback = [e for e in episodes if e.was_downloaded(and_exists=True)]
        to_download = [e for e in episodes if e not in to_playback]

        if to_playback:
            self._playback_episodes(to_playback)

        if to_download:
            self._download_episodes(to_download)

        return (1, 'Success')

    @dbus.service.method(dbus_interface=gpodder.dbus_podcasts, in_signature='', out_signature='')
    def check_for_updates(self):
        """Check for new episodes or offer subscriptions"""
        self._on_check_for_updates()


import json
import os
import sys
import asyncio
import pathlib
import websockets
import logging

from vosk import BatchModel, BatchRecognizer, GpuInit

async def recognize(websocket, path):
    global args
    global pool
    global model

    rec = None
    sample_rate = args.sample_rate

    logging.info(f'Connection from {websocket.remote_address}');

    while True:

        message = await websocket.recv()

        # Load configuration if provided
        if isinstance(message, str) and 'config' in message:
            jobj = json.loads(message)['config']
            logging.info("Config %s", jobj)
            if 'sample_rate' in jobj:
                sample_rate = float(jobj['sample_rate'])
            continue

        # Create the recognizer, word list is temporary disabled since not every model supports it
        if not rec:
            rec = BatchRecognizer(model, sample_rate)

        if message == '{"eof" : 1}':
            rec.FinishStream()
            break

        if isinstance(message, str) and 'config' in message:
            continue

        rec.AcceptWaveform(message)

        while rec.GetPendingChunks() > 0:
            await asyncio.sleep(0.1)

        res = rec.Result()
        if len(res) == 0:
            await websocket.send('{ "partial" : "" }')
        else:
            await websocket.send(res)

    while rec.GetPendingChunks() > 0:
        await asyncio.sleep(0.1)

    res = rec.Result()
    await websocket.send(res)

async def start():

    global model
    global args

    # Enable loging if needed
    #
    # logger = logging.getLogger('websockets')
    # logger.setLevel(logging.INFO)
    # logger.addHandler(logging.StreamHandler())
    logging.basicConfig(level=logging.INFO)

    args = type('', (), {})()

    args.interface = os.environ.get('VOSK_SERVER_INTERFACE', '0.0.0.0')
    args.port = int(os.environ.get('VOSK_SERVER_PORT', 2700))
    args.sample_rate = float(os.environ.get('VOSK_SAMPLE_RATE', 8000))

    GpuInit()

    model = BatchModel()

    async with websockets.serve(recognize, args.interface, args.port):
        await asyncio.Future()

if __name__ == '__main__':
    asyncio.run(start())


import numpy as np
import pandas as pd
import torch
import torch.nn as nn

# read csv file
df = pd.read_csv('housing.csv')
print(df.head())

df.columns = np.array(range(0, len(df.columns)))
print(df.head())

# Train data

x_train = df.iloc[:, range(0, 3)]
x_train = torch.FloatTensor(x_train.values)
print(x_train)

y_train = df.iloc[:, 3]
y_train = torch.FloatTensor(y_train.values)
y_train = y_train.view(y_train.size(0), 1)
print(y_train)

# Normalization

x_mean = torch.mean(x_train)
x_std = torch.std(x_train)
x_train -= x_mean
x_train /= x_std
print(x_train)

y_mean = torch.mean(y_train)
y_std = torch.std(y_train)
y_train -= y_mean
y_train /= y_std
print(y_train)

# Model

class Model(nn.Module):
    def __init__(self):
        super().__init__()
        self.input = nn.Linear(3, 128)
        self.layer1 = nn.Sequential(
            nn.Linear(128, 128),
            nn.ReLU()
        )
        self.layer2 = nn.Sequential(
            nn.Linear(128, 64),
            nn.ReLU()
        )
        self.layer3 = nn.Sequential(
            nn.Linear(64, 32),
            nn.ReLU()
        )
        self.output = nn.Linear(32, 1)

    def forward(self, x):
        out = self.input(x)
        out = self.layer1(out)
        out = self.layer2(out)
        out = self.layer3(out)
        out = self.output(out)
        return out

model = Model()
criterion = nn.MSELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0.9)

# Training

for epoch in range(1000):
    prediction = model(x_train)
    cost = criterion(prediction, y_train)

    optimizer.zero_grad()
    cost.backward()
    optimizer.step()

    if epoch % 10 == 0:
        print('Epoch : {}, cost : {:.6f}'.format(epoch, cost.item()))

# Prediction

new_var = torch.FloatTensor([[-0.8156, -1.2938,  0.7935]])
print(model(new_var))


## if the reward is beyond the threshold, then start randomizing the initial position

from stable_baselines3.common.callbacks import BaseCallback
from stable_baselines3.common.utils import safe_mean

class StartRandomizingCallback(BaseCallback):
    """
    平均報酬が閾値を超えたら、初期位置のランダム化を開始するコールバック。
    """
    def __init__(self, threshold, env, model, verbose=0):
        super(StartRandomizingCallback, self).__init__(verbose)
        self.threshold = threshold
        self.changed = False
        self.ep_rew_mean = None
        self.env = env
        self.model = model

    def _on_step(self) -> bool:
        """
        This method is called before collecting the rollouts.
        """
        self.ep_rew_mean = safe_mean([ep_info["r"] for ep_info in self.model.ep_info_buffer])
        if self.ep_rew_mean and (self.ep_rew_mean > self.threshold) and not self.changed:
            self.training_env.env_method("start_randomizing_position")
            self.changed = True

        return True


from pymongo import MongoClient
from creds import db_string
import certifi
import sys

ca = certifi.where()

try:

    db_client = MongoClient(db_string, tlsCAFile=ca)
    db = db_client.test
    # print("Connection successful")

except Exception as e:
    print(str(e))
    print("<< Probably bad Internet or IP is not allowed to connect >>")
    sys.exit()

try:
    current_db = db_client["vuln"]
    scan_collection = current_db["scan"]
    lut_collection = current_db["lut"]

except Exception as e:
    print(str(e))
    sys.exit()


import urllib.request,json
from .models import Quote

api = None

def configure_request(app):
    global api
    api = app.config['API']

def get_quotes():
    '''
    Function that gets the json response to our url request
    '''
    with urllib.request.urlopen(api) as url:
        get_quote = url.read()
        get_quote_response = json.loads(get_quote)

        quote_results = []

        author = get_quote_response.get('author')
        quote = get_quote_response.get('quote')

        if author and quote:
            quote_object = Quote(author,quote)
            quote_results.append(quote_object)

    return quote_results


import datetime
import time
import cv2
import numpy as np
import threading

# initialize the video stream and allow the cammera sensor to warmup
vs = cv2.VideoCapture(0)
time.sleep(2.0)
avg=[0,0]
count=0
x = []
y = []

def imageStream():
    # loop over the frames from the video stream
    while True:
        (grabbed, frame) = vs.read()

        # draw the timestamp on the frame
        timestamp = datetime.datetime.now()
        ts = timestamp.strftime("%A %d %B %Y %I:%M:%S%p")
        cv2.putText(frame, ts, (10, frame.shape[0] - 10), cv2.FONT_HERSHEY_SIMPLEX,0.90, (255, 255, 255), 1)
        cv2.circle(frame,(320,240),25,(0,255,),3)
        # show the frame
        cv2.imshow("Nozzle", frame)
        key = cv2.waitKey(10)

def userInterface():
    print("")
    tc=input("Press enter to close the window when done. \n")

#######################
# Main Code Starts Here
#######################
th = threading.Thread(target=imageStream)
th.daemon = True # Thread will be killed when main exits.
th.start()

userInterface()


"""
Created on Sat Sep 18 23:11:22 2021

@author: datakind
"""
import logging
import os
import sys
import typing as T
from functools import reduce
from pathlib import Path

import pandas as pd
import requests
from matplotlib import collections
from matplotlib import pyplot as plt

from analysis.acs_correlation import correlation_analysis
from analysis.acs_data import get_acs_data
from analysis.housing_loss_summary import summarize_housing_loss
from analysis.timeseries import create_timeseries
from collection.address_cleaning import remove_special_chars
from collection.address_geocoding import find_state_county_city, geocode_input_data
from collection.address_validation import (
    standardize_input_addresses,
    validate_address_data,
    verify_input_directory,
)
from collection.tigerweb_api import (
    create_tigerweb_query,
    get_input_data_geometry,
    jprint,
    rename_baseline,
)
from const import (
    ACS_DATA_DICT_FILENAME,
    ACS_YEAR,
    GEOCODED_EVICTIONS_FILENAME,
    GEOCODED_FORECLOSURES_FILENAME,
    GEOCODED_TAX_LIENS_FILENAME,
    GIS_IMPORT_FILENAME,
    HOUSING_LOSS_SUMMARY_FILENAME,
    HOUSING_LOSS_TIMESERIES_FILENAME,
    MAX_YEAR,
    MIN_YEAR,
    OUTPUT_ALL_HOUSING_LOSS_PLOTS,
    OUTPUT_EVICTION_PLOTS,
    OUTPUT_FORECLOSURE_PLOTS,
    OUTPUT_PATH_GEOCODED_DATA,
    OUTPUT_PATH_GEOCODER_CACHE,
    OUTPUT_PATH_MAPS,
    OUTPUT_PATH_PLOTS,
    OUTPUT_PATH_PLOTS_DETAIL,
    OUTPUT_PATH_SUMMARIES,
    TRACT_BOUNDARY_FILENAME,
    EVIC_ADDRESS_ERR_FILENAME,
    MORT_ADDRESS_ERR_FILENAME,
    TAX_ADDRESS_ERR_FILENAME
)

def load_data(sub_directories: T.List, data_category) -> T.Tuple[pd.DataFrame,pd.DataFrame]:
    """Load evictions data from csv template
    Inputs
    ------
    sub_directories: list of sub-directories
    data_category: 'evictions', 'mortgage_foreclosures', 'tax_lien_foreclosures'
    parameters: If necessary, parameters to determine narrow down timeframe
      or columns of evictions data to return
    Outputs
    -------
    cleaned_df: Processed pandas dataframe for next step of geo matching
    duplicate_nan_df: Addresses that are dropped for being duplicates or nan for record keeping
    """
    for data_dir in sub_directories:
        # If this sub directory does not match the data_category, skip it:
        if data_category not in str(data_dir):
            continue
        # If this is right subdirectory, list the files in the directory
        data_files = os.listdir(data_dir)
        # Alert user if there are no files in the relevant subdirectory
        if len(data_files) == 0:
            print('\n\u2326', 'Your folder ', data_dir, 'is empty. Please add your', data_category ,'dataset to this folder.')
            return None, None
        else:
            print(
                '\nSubdirectory of ',
                data_dir,
                ' has ',
                len(data_files),
                ' files in it: ',
                data_files,
            )
        data = pd.DataFrame()
        # Loop through the files
        for f in data_files:
            print('Loading file: ', f, ' of ', data_files)
            # Read in file depending on file format
            if str(f.lower()).startswith(data_category):
                if str(f.lower()).endswith('.csv'):
                    print(u'\u2713', 'File type: .csv')
                    df = pd.read_csv(data_dir / f, low_memory=False)
                    print('First row of data:\n', df.iloc[0, :])
                elif str(f.lower()).endswith(('.xls', '.xlsx')):
                    print(u'\u2713', 'File type: .xls or .xlsx')
                    df = pd.read_excel(data_dir / f)
                    print(u'\u2713', 'First row of data:\n', df.iloc[0, :])
                else:
                    print(f'Invalid file detected {str(f)}')
            else:
                # Let user know about invalid files
                print(
                    u'\u2326',
                    'A file that was not labeled as ',
                    data_category,
                    ' was found and will be ignored.',
                )
                print(
                    u'\u2326',
                    'Please name each type of file according to the guidelines.',
                )
                continue
            # Join multiple files together if there are multple, otherwise move on
            if len(df) == 0:
                data = df
            else:
                data = pd.concat([data, df], ignore_index=True)
        # No files with readable extensions
        if len(data) == 0:
            print(
                u'\u2326',
                'No readable files found in sub-directory!',
                'Please make sure input file is CSV.',
            )
            return None, None
        else:
            print(
                u'\u2713',
                'You have at least one data file to process of type: ',
                data_category,
            )
        print('\nProcessing duplicate and empty rows:')
        # Drop FULL duplicates
        rows = data.shape[0]
        print('You are starting with ', rows, ' rows in your data set.')
        # Convert columns names to lowercase and remove any special characters
        data.columns = [
            remove_special_chars(col.replace(' ', "_").lower().strip())
            for col in data.columns
        ]
        if ('street_address_1' in data.columns and 'city' in data.columns and 'state' in data.columns and 'zip_code' in data.columns):
           #select records that na for street_address_1 or all fields as df_dups_na
            df_dups_na = data[data['street_address_1'].isna()]
            df_dups_dups = data[data.duplicated()]
            df_dups_na['errors'] = 'NA'
            df_dups_dups['errors'] = 'Duplicate'
            df_dups_out_na = df_dups_na[['street_address_1','city', 'state', 'zip_code', 'errors']]
            df_dups_out_dups = df_dups_dups[['street_address_1', 'city', 'state', 'zip_code', 'errors']]
            df_dups_out = pd.concat([df_dups_out_na, df_dups_out_dups])
        else:
            print(
                'You are missing one of the following required column: street_address_1, city, state, or zip_code.'
            )
            return None, None
        data = data.drop_duplicates().dropna(how="all", axis=0)
        if ('street_address_1' in data.columns):
            data = data.dropna(subset=['street_address_1'])
        print(
            u'\u2326',
            'Dropping duplicates and null rows removed ',
            round(abs(100 * (data.shape[0] - rows) / rows), 1),
            '% of your rows.',
        )

        print('\nProcessing Date Columns:')
        if data_category == 'evictions':
            date_column = 'eviction_filing_date'
        elif data_category == 'mortgage_foreclosures':
            date_column = 'foreclosure_sale_date'
        elif data_category == 'tax_lien_foreclosures':
            date_column = 'tax_lien_sale_date'
        # Check to see if they have the correct date column
        # but otherwise use the first 'date' column found:
        has_date_column = date_column in data.columns
        if not has_date_column:
            print(u'\u2326', 'Expected date column is missing in the input file')
            date_columns_found = [
                item for item in data.columns if 'date' in item.lower()
            ]
            if len(date_columns_found) > 0:
                print(
                    u'\u2713',
                    'Process will use ',
                    date_columns_found[0],
                    ' as the ',
                    date_column,
                )
                data[date_column] = data[date_columns_found[0]].copy()
            else:
                print(
                    u'\u2326',
                    'Date column not found, please adjust your column headers.',
                )
                return None, None
        else:
            print(f'\u2713  Process will use {date_column}.')
        # If date column is null, tell user to fix the file

        if data[date_column].isnull().all():
            print(
                u'\u2326',
                'Date column is empty - please ensure date data is included.',
            )
            return None, None

        if data[date_column].dtype == object:
            data[date_column] = pd.to_datetime(
                data[date_column], infer_datetime_format=True
            )
        # Deal with null dates
        if data[date_column].isnull().sum() / data.shape[0] > 0.25:
            print(
                'The percent of the date column that is null is: ',
                round(100 * data[date_column].isnull().sum() / data.shape[0], 1),
                '%, which can negatively affect the time-series analysis.',
            )

        data[date_column] = pd.to_datetime(data[date_column])
        data[date_column] = data[date_column].fillna(method='ffill')
        print(f'\nFiltering data to only >= {MIN_YEAR} values:')
        # Create year and month columns for later aggregation
        data['year'] = data[date_column].dt.year.astype(int)
        data['month'] = data[date_column].dt.to_period('M').astype(str)
        print(
            u'\u2713',
            'Data date range is from ',
            data[date_column].dt.date.min(),
            ' to ',
            data[date_column].dt.date.max(),
        )
        # Let user know how much older data will be be discarded
        if data.year.min() < MIN_YEAR:
            print(
                u'\u2326',
                'Data before',
                MIN_YEAR,
                'represents',
                round(
                    100
                    * data[(data.year < MIN_YEAR) | (data.year > MAX_YEAR)].shape[0]
                    / data.shape[0],
                    1,
                ),
                '% of data and cannot be used in this analysis.',
            )
        print(u'\u2713', 'Date column has been processed.\n')
        data = data[(data.year >= MIN_YEAR) & (data.year <= MAX_YEAR)]
        print(u'\u2713', 'Data loading complete. Address validation is next.')

        return data, df_dups_out

    print(
        u'\u2326',
        'No', data_category, 'sub-folder exists in the folder.',
        'Please add a', data_category, 'sub-folder.',
        'Please review FLH Partner Site Data Collection Template for more details.',
    )
    return None, None

def write_df_to_disk(input_df: pd.DataFrame, write_path_filename: Path) -> None:
    """Simple helper function to write a dataframe to disk."""
    # Check for empty input
    if input_df is None:
        return None
    if write_path_filename.parent is None or not write_path_filename.parent.exists():
        print(f"Invalid output directory for input dataframe {input_df}")
        return None
    input_df.to_csv(str(write_path_filename), index=False)

def main(input_path: str) -> None:
    """This function is what it says it is. :)

    It takes in the input data path as an argument
    """
    # LOOK FOR CORRECT SUBDIRECTORY STRUCTURE
    sub_directories = verify_input_directory(input_path)
    # If the input_directory fails, the main function should abort:
    if sub_directories is None:
        return("The path provided does not include the following three folders: evictions, mortgage_foreclosures, and tax_lien_foreclosures. ",
        'Please add these three files to the folder to proceed')

    # LOAD ALL 3 TYPES OF DATA (AS AVAILABLE)
    df_evic, df_evic_dups = load_data(sub_directories, 'evictions')
    df_mort, df_mort_dups = load_data(sub_directories, 'mortgage_foreclosures')
    df_tax, df_tax_dups = load_data(sub_directories, 'tax_lien_foreclosures')

    if (df_evic is None) and (df_mort is None) and (df_tax is None):
        print(
            'No data files matched our requirements for this analysis.'
            'Please check the files and restart.'
        )
        return None

    # STANDARDIZE THE INPUT DATA ADDRESSES
    df_evic_standardized, df_evic_parse_err, evic_avail_cols = standardize_input_addresses(
        df_evic, 'eviction'
    )
    df_mort_standardized, df_mort_parse_err, mort_avail_cols = standardize_input_addresses(
        df_mort, 'foreclosure'
    )
    df_tax_standardized, df_tax_parse_err, tax_avail_cols = standardize_input_addresses(
        df_tax, 'tax lien'
    )

    # CREATE TIME SERIES PLOTS
    plt.rcParams['figure.figsize'] = [25, 10]
    fig1 = create_timeseries(df_evic_standardized, 'eviction_filing_date', 'Evictions')
    fig2 = create_timeseries(
        df_mort_standardized, 'foreclosure_sale_date', 'Foreclosures'
    )
    fig3 = create_timeseries(df_tax_standardized, 'tax_lien_sale_date', 'Tax_Liens')
    # Create the directories to output the plots to
    plot_write_path = Path(input_path).parent / OUTPUT_PATH_PLOTS
    plot_write_path.mkdir(parents=True, exist_ok=True)
    # Save the plots to this directory
    plt.legend(prop={'size': 20})
    plt.savefig(str(plot_write_path / HOUSING_LOSS_TIMESERIES_FILENAME))
    print(
        '*** Created housing loss timeseries image '
        + str(plot_write_path / HOUSING_LOSS_TIMESERIES_FILENAME)
    )

    # GEOCODE THE CLEANED/STANDARDIZED DATA AND WRITE GEOCODED DATASETS TO DISK
    # Create the directories to output the geocoder cache to
    geocoder_cache_write_path = Path(input_path).parent / OUTPUT_PATH_GEOCODER_CACHE
    geocoder_cache_write_path.mkdir(parents=True, exist_ok=True)

    df_evic_geocoded_final = None
    df_mort_geocoded_final = None
    if df_evic_standardized is not None:
        df_evic_geocoded_final = geocode_input_data(
            df_evic_standardized, evic_avail_cols, 'eviction', geocoder_cache_write_path
        )
    if df_mort_standardized is not None:
        df_mort_geocoded_final = geocode_input_data(
            df_mort_standardized,
            mort_avail_cols,
            'foreclosure',
            geocoder_cache_write_path,
        )
    df_tax_geocoded_final = geocode_input_data(
        df_tax_standardized, tax_avail_cols, 'tax lien', geocoder_cache_write_path
    )

    # Create the directories to output the raw geocoded datasets to
    geocoded_file_write_path = Path(input_path).parent / OUTPUT_PATH_GEOCODED_DATA
    geocoded_file_write_path.mkdir(parents=True, exist_ok=True)

    write_df_to_disk(
        df_evic_geocoded_final, geocoded_file_write_path / GEOCODED_EVICTIONS_FILENAME
    )
    write_df_to_disk(
        df_mort_geocoded_final,
        geocoded_file_write_path / GEOCODED_FORECLOSURES_FILENAME,
    )
    write_df_to_disk(
        df_tax_geocoded_final, geocoded_file_write_path / GEOCODED_TAX_LIENS_FILENAME
    )

    # Get the most likely state/county FIPS codes and city from geocoded data
    state_fips, county_fips, city_str, state_str = find_state_county_city(
        df_evic_geocoded_final
    )
    if state_fips is None or county_fips is None:
        # If we can't find FIPS codes from evictions data, try it from the mortgage foreclosure data
        state_fips, county_fips, city_str_2, state_str_2 = find_state_county_city(
            df_mort_geocoded_final
        )

    # GRAB ACS DATA; used in housing loss summary and demographic correlation search
    print("\nPreparing to get ACS data...")
    acs_df = pd.DataFrame()
    for county in county_fips:
        acs_df_county, acs_data_dict = get_acs_data(state_fips, str(county), ACS_YEAR)
        acs_df = pd.concat([acs_df, acs_df_county], axis=0)
    if acs_df is None:
        print(
            '\u2326  Insufficient geography information to retrieve ACS Data!',
            'Please input valid state and county FIPS codes.',
        )
        return None

    # Create the directories to output the ACS data and summary files to
    summary_write_path = Path(input_path).parent / OUTPUT_PATH_SUMMARIES
    summary_write_path.mkdir(parents=True, exist_ok=True)

    # Write the ACS data dictionary to a file for reference
    if acs_data_dict is not None:
        pd.DataFrame(acs_data_dict).transpose().reset_index().rename(
            columns={"index": "variable"}
        ).to_csv(str(summary_write_path / ACS_DATA_DICT_FILENAME), index=False)
        print(
            '*** Created '
            + str(summary_write_path / ACS_DATA_DICT_FILENAME)
            + ' - inspect for ACS variable definitions and reference'
        )

    ### Grab the renter and home owner total count estimates we'll use
    ### later for housing loss rate calculations
    renter_hhs = acs_df[['GEOID', 'total-renter-occupied-households']].copy()
    owner_hhs = acs_df[['GEOID', 'total-owner-occupied-households']].copy()
    # Standardize column name
    renter_hhs.rename(
        columns={'total-renter-occupied-households': 'households_by_geoid'},
        inplace=True,
    )
    owner_hhs.rename(
        columns={'total-owner-occupied-households': 'households_by_geoid'}, inplace=True
    )

    #Get the exceptions from geocoded data- those that dont merge with the acs data
    df_evic_errors = None
    df_mort_errors = None
    df_tax_errors = None
    if df_evic_geocoded_final is not None:
        geoid_noacs_evic = df_evic_geocoded_final.merge(renter_hhs, left_on='geoid', right_on='GEOID', how = 'left', indicator=True)
        geoid_noacs_evic = geoid_noacs_evic[geoid_noacs_evic['_merge'] == 'left_only'][['street_address_1', 'city', 'state', 'zip_code']]
        no_geoid_evic = df_evic_geocoded_final[df_evic_geocoded_final['geoid'].isna()][['street_address_1', 'city', 'state', 'zip_code']]
        no_geoid_evic['errors'] = 'Unable to find a match in the census geocoder'
        df_evic_errors = no_geoid_evic
    if df_mort_geocoded_final is not None:
        geoid_noacs_mort = df_mort_geocoded_final.merge(owner_hhs, left_on='geoid', right_on='GEOID', how = 'left', indicator=True)
        geoid_noacs_mort = geoid_noacs_mort[geoid_noacs_mort['_merge'] == 'left_only'][['street_address_1', 'city', 'state', 'zip_code']]
        no_geoid_mort = df_mort_geocoded_final[df_mort_geocoded_final['geoid'].isna()][['street_address_1', 'city', 'state', 'zip_code']]
        no_geoid_mort['errors'] = 'Unable to find a match in the census geocoder'
        df_mort_errors = no_geoid_mort
    if df_tax_geocoded_final is not None:
        geoid_noacs_tax = df_tax_geocoded_final.merge(owner_hhs, left_on='geoid', right_on='GEOID', how = 'left', indicator=True)
        geoid_noacs_tax = geoid_noacs_tax[geoid_noacs_tax['_merge'] == 'left_only'][['street_address_1', 'city', 'state', 'zip_code']]
        no_geoid_tax = df_tax_geocoded_final[df_tax_geocoded_final['geoid'].isna()][['street_address_1', 'city', 'state', 'zip_code']]
        no_geoid_tax['errors'] = 'Unable to find a match in the census geocoder'
        df_tax_errors = no_geoid_tax

    # CREATE HOUSING LOSS SUMMARIES
    evic_summ = summarize_housing_loss(df_evic_geocoded_final, renter_hhs, 'evic')
    mort_summ = summarize_housing_loss(df_mort_geocoded_final, owner_hhs, 'mort')
    tax_summ = summarize_housing_loss(df_tax_geocoded_final, owner_hhs, 'tax')

    # Stack the data together for summarization while counting all housing loss events for the housing loss index calculation
    coll_dfs = []
    msg = ' with data from'
    if evic_summ is not None:
        coll_dfs.append(evic_summ)
        msg += ' evictions;'
    if mort_summ is not None:
        coll_dfs.append(mort_summ)
        msg += ' mortgage foreclosures;'
    if tax_summ is not None:
        coll_dfs.append(tax_summ)
        msg += ' tax liens'

    # Create the summary dataframe, add the ACS variables and get all housing loss events
    df_summ_mrg = reduce(
        lambda left, right: pd.merge(left, right, on='geoid'), coll_dfs
    )
    df_summ_mrg = pd.merge(df_summ_mrg, acs_df, left_on='geoid', right_on='GEOID')

    if mort_summ is not None:
        if tax_summ is not None:
            df_summ_mrg['total_housing_loss'] = (
                df_summ_mrg['total_filings']
                + df_summ_mrg['total_mortgage_foreclosures']
                + df_summ_mrg['total_tax_liens']
            )
            df_summ_mrg['total_foreclosures'] = (
                df_summ_mrg['total_mortgage_foreclosures']
                + df_summ_mrg['total_tax_liens']
            )
        else:
            df_summ_mrg['total_housing_loss'] = (
                df_summ_mrg['total_filings']
                + df_summ_mrg['total_mortgage_foreclosures']
            )
            df_summ_mrg['total_foreclosures'] = df_summ_mrg[
                'total_mortgage_foreclosures'
            ]
    else:
        if tax_summ is not None:
            df_summ_mrg['total_housing_loss'] = (
                df_summ_mrg['total_filings'] + df_summ_mrg['total_tax_liens']
            )
            df_summ_mrg['total_foreclosures'] = df_summ_mrg['total_tax_liens']
        else:
            df_summ_mrg['total_housing_loss'] = df_summ_mrg['total_filings']

    # Calculate the housing loss index - note this assumes complete records of evictions and foreclosures for all years present in eviction data
    df_summ_mrg['housing-loss-index'] = (
        df_summ_mrg['total_housing_loss']
        / (
            df_summ_mrg['total-renter-occupied-households']
            + df_summ_mrg['total-owner-occupied-households']
        )
        / df_summ_mrg['nyears_evic_data']
    )

    # Save the summary file to this directory
    write_df_to_disk(df_summ_mrg, summary_write_path / HOUSING_LOSS_SUMMARY_FILENAME)
    print(
        '*** Created ' + str(summary_write_path / HOUSING_LOSS_SUMMARY_FILENAME) + msg
    )

    #Create summary of the errors and output to file
    if (df_evic_parse_err is not None or df_evic_errors is not None or df_evic_dups is not None):
        df_evic_errors = pd.concat([df_evic_parse_err, df_evic_errors, df_evic_dups])
        write_df_to_disk(
            df_evic_errors, summary_write_path / EVIC_ADDRESS_ERR_FILENAME
        )
    if (df_mort_parse_err is not None or df_mort_errors is not None or df_mort_dups is not None):
        df_mort_errors = pd.concat([df_mort_parse_err,df_mort_errors, df_mort_dups])
        write_df_to_disk(
            df_mort_errors, summary_write_path / MORT_ADDRESS_ERR_FILENAME
        )
    if (df_tax_parse_err is not None or df_tax_errors is not None or df_tax_dups is not None):
        df_tax_errors = pd.concat([df_tax_parse_err, df_tax_errors, df_tax_dups])
        write_df_to_disk(
            df_tax_errors, summary_write_path / TAX_ADDRESS_ERR_FILENAME
        )

    # Prepare subdirectories to store correlation analysis results

    all_housing_loss_write_path = (
        Path(input_path).parent / OUTPUT_ALL_HOUSING_LOSS_PLOTS
    )
    all_housing_loss_write_path.mkdir(parents=True, exist_ok=True)

    all_housing_loss_plot_detail_write_path = (
        Path(input_path).parent
        / OUTPUT_ALL_HOUSING_LOSS_PLOTS
        / OUTPUT_PATH_PLOTS_DETAIL
    )
    all_housing_loss_plot_detail_write_path.mkdir(parents=True, exist_ok=True)

    evictions_write_path = Path(input_path).parent / OUTPUT_EVICTION_PLOTS
    evictions_write_path.mkdir(parents=True, exist_ok=True)

    evictions_plot_detail_write_path = (
        Path(input_path).parent / OUTPUT_EVICTION_PLOTS / OUTPUT_PATH_PLOTS_DETAIL
    )
    evictions_plot_detail_write_path.mkdir(parents=True, exist_ok=True)

    foreclosure_write_path = Path(input_path).parent / OUTPUT_FORECLOSURE_PLOTS
    foreclosure_write_path.mkdir(parents=True, exist_ok=True)

    foreclosure_plot_detail_write_path = (
        Path(input_path).parent / OUTPUT_FORECLOSURE_PLOTS / OUTPUT_PATH_PLOTS_DETAIL
    )
    foreclosure_plot_detail_write_path.mkdir(parents=True, exist_ok=True)

    # # RUN CORRELATION ANALYSIS WITH ACS VARIABLES
    # target_var options:
    # evictions --> 'total_filings'
    # mortgage foreclosures including tax liens --> 'total_foreclosures'
    # all housing loss events --> 'housing-loss-index'
    plt.rcParams['figure.figsize'] = [15, 10]

    try:
        correlation_analysis(
            acs_df, df_summ_mrg, 'housing-loss-index', all_housing_loss_write_path
        )
    except KeyError:
        print('Unable to create correlations for housing-loss-index')

    try:
        correlation_analysis(acs_df, df_summ_mrg, 'total_filings', evictions_write_path)
    except KeyError:
        print('Unable to create correlations for total_filings')

    try:
        correlation_analysis(
            acs_df, df_summ_mrg, 'total_foreclosures', foreclosure_write_path
        )
    except KeyError:
        print('Unable to create correlations for total_foreclosures')

    # Create the directories to output the mapping files to
    mapping_write_path = Path(input_path).parent / OUTPUT_PATH_MAPS
    mapping_write_path.mkdir(parents=True, exist_ok=True)

    # GET GEOMETRY DATA FROM CENSUS TIGERWEB API
    print("\nRetrieving geography data from Census TIGERweb API...")
    geojson_gdf = get_input_data_geometry(
        state_fips, county_fips, str(mapping_write_path / TRACT_BOUNDARY_FILENAME)
    )
    print('*** Created ' + str(mapping_write_path / TRACT_BOUNDARY_FILENAME))
    # Merge the geometry dataframe with the housing + ACS data summary, but avoid
    #   duplicate column names (since they are non-case sensitive in databases)
    # First drop the 'index' column also, since the `censusdata.censusgeo.censusgeo` datatype
    #   throws an error in the .gpkg file creation
    df_summ_mrg.drop(columns='index', inplace=True)
    if 'geoid' in df_summ_mrg.columns and 'GEOID' in df_summ_mrg.columns:
        df_summ_mrg.drop(columns='GEOID', inplace=True)
    elif 'GEOID' in df_summ_mrg.columns:
        df_summ_mrg.rename(columns={'GEOID': 'geoid'}, inplace=True)
    df_summ_mrg['geoid'] = df_summ_mrg['geoid'].astype('str').str.zfill(11)
    merged_gdf = geojson_gdf.merge(df_summ_mrg, how='left', on='geoid')
    merged_gdf.to_file(str(mapping_write_path / GIS_IMPORT_FILENAME), driver='GPKG')
    print('*** Created ' + str(mapping_write_path / GIS_IMPORT_FILENAME))

    # Now that we have got through the entire process, delete the cached geocoded files
    for f in geocoder_cache_write_path.iterdir():
        if f.is_file():
            f.unlink()
    geocoder_cache_write_path.rmdir()

    return None

if __name__ == "__main__":
    logging.basicConfig(
        level=logging.INFO, format="%(asctime)s - %(levelname)s: %(message)s"
    )
    dir_path = sys.argv[1]
    main(dir_path)


from fastapi import APIRouter, Depends
from core import notification_controller
from core.base import mssg_response
from fastapi.security import HTTPBearer
from fastapi_jwt_auth import AuthJWT

dbn = notification_controller.DBNotificationController()
router = APIRouter()

# need for docs auth button
token_auth_scheme = HTTPBearer()

@router.post('/notify/email', tags=['notification'])
def send_email(
        reply_to: str,
        attach: str,
        subject: str = 'Event between <user> and <user>',
        message: str = '<user> requested you for meeting, details:',
        Authorize: AuthJWT = Depends(), token=Depends(token_auth_scheme)
    ):
    Authorize.jwt_required()
    return mssg_response(dbn.send_email(reply_to, subject, message, attach))


import ast
import importlib
import json

import databases
import tenacity

from _clients.participants import ledger
from _utils import db_utils, utils
import os
import signal

class Participant:
    """
    Participant is the interface layer between local resources and the Market
    """
    def __init__(self, sio_client, participant_id, market_id, db_path, trader_params, storage_params, **kwargs):
        # Initialize participant variables
        self.server_online = False
        self.run = True
        self.market_id = market_id
        self.market_connected = False
        self.participant_id = str(participant_id)
        self.__client = sio_client
        self.client = sio_client
        self.__profile = {
            'db_path': db_path
        }

        # Initialize market variables
        self.__ledger = ledger.Ledger(self.participant_id)
        self.__extra_transactions = {}
        self.__market_info = {}
        self.__meter = {}
        self.__timing = {}

        # Initialize trader variables and functions
        trader_params = json.loads(trader_params)
        trader_fns = {
            'id': self.participant_id,
            'timing': self.__timing,
            'ledger': self.__ledger,
            'extra_transactions': self.__extra_transactions,
            'market_info': self.__market_info,
            'read_profile': self.__read_profile,
            'get_profile_stats': self.__get_profile_stats,
            'meter': self.__meter
        }

        if storage_params:
            storage_params = json.loads(storage_params)
            storage_type = storage_params.pop('type', None)
            # self.storage_fns = {
            #     'id': self.participant_id,
            #     'timing': self.__timing
            # }
            self.storage = importlib.import_module('_devices.' + storage_type).Storage(**storage_params)
            self.storage.timing = self.__timing
            trader_fns['storage'] = {
                'info': self.storage.get_info,
                'check_schedule': self.storage.check_schedule,
                # 'schedule_energy': self.storage.schedule_energy
            }

        trader_type = trader_params.pop('type', None)
        if trader_type == 'remote_agent':
            trader_fns['emit'] = self.__client.emit
        Trader = importlib.import_module('_agent.traders.' + trader_type).Trader
        self.trader = Trader(trader_fns=trader_fns, **trader_params)

        self.__profile_params = {
            'generation_scale': kwargs['generation_scale'] if 'generation_scale' in kwargs else 1,
            'load_scale': kwargs['load_scale'] if 'load_scale' in kwargs else 1
        }
        synthetic_profile = trader_params.pop('use_synthetic_profile', None)
        if synthetic_profile:
            self.__profile_params['synthetic_profile'] = synthetic_profile

        # if 'market_ns' in kwargs:
        #     NSMarket = importlib.import_module(kwargs['market_ns']).NSMarket
            # self.__client.register_namespace(NSMarket(self))

    async def delay(self, s):
        await self.__client.sleep(s)

    # async def disconnect(self):
    #     '''
    #     This method disconnects the client from the server
    #     '''
    #     await self.__client.disconnect()

    async def open_db(self, db_path):
        """Opens connection to the database where load and generation profiles are stored.
        Also stores references to the database object and the table object

        Args:
            db_path ([type]): [description]
        """
        self.__profile['db_path'] = db_path
        self.__profile['db'] = databases.Database(db_path)
        profile_name = self.__profile_params['synthetic_profile'] if 'synthetic_profile' in self.__profile_params\
            else self.participant_id
        self.__profile['name'] = profile_name
        self.__profile['db_table'] = db_utils.get_table(db_path, profile_name)
        if 'db_table' in self.__profile or self.__profile['db_table'] is not None:
            await self.__profile['db'].connect()

    async def __get_profile_stats(self):
        """reads and returns pre-calculated profile statistics for calculating Z scores, if available.
        """
        db = self.__profile['db']
        table = db_utils.get_table(self.__profile['db_path'], "_statistics")
        query = table.select().where(table.c.name == self.__profile['name'])
        # async with db.transaction():
        row = await db.fetch_one(query)
        if row is not None:
            return dict(row)
        return None

    async def open_profile_db(self):
        await self.open_db(self.__profile['db_path'])
        # await self.get_profile_stats(self.__profile['db_path'])

    async def join_market(self):
        """Emits event to join a Market
        """
        client_data = {
            'type': ('participant', 'Residential'),
            'id': self.participant_id,
            'market_id': self.market_id
        }
        await self.__client.emit('join_market', client_data, callback=self.register_success)

    # Continuously attempt to join server
    async def register_success(self, success):
        if not success:
            # self.__profiles_available()
            await self.delay(3)
            await self.join_market()
        self.busy = False

    async def update_extra_transactions(self, message):
        time_delivery = tuple(message.pop('time_delivery'))
        self.__ledger.extra[time_delivery] = message
        self.__extra_transactions.clear()
        self.__extra_transactions.update(message)

    @tenacity.retry(wait=tenacity.wait_random(0, 3))
    async def bid(self, time_delivery=None, **kwargs):
        """Submit a bid

        Args:
            time_delivery ([type], optional): [description]. Defaults to None.
        """

        # quantity is energy in Wh
        # price is $/kWh
        if time_delivery is None:
            time_delivery = self.__timing['next_settle']

        bid_entry = {
            'participant_id': self.participant_id,
            'quantity': kwargs['quantity'],  # Wh
            'price': kwargs['price'],  # $/kWh
            'time_delivery': time_delivery
        }
        # print('bidding', self.trader.is_learner, self.__timing, bid_entry)
        await self.__client.emit('bid', bid_entry)

    @tenacity.retry(wait=tenacity.wait_random(0, 3))
    async def ask(self, time_delivery=None, **kwargs):
        """Submit an ask

        Args:
            time_delivery ([type], optional): [description]. Defaults to None.
        """
        # quantity is energy in Wh
        # price is $/kWh
        if time_delivery is None:
            time_delivery = self.__timing['next_settle']

        ask_entry = {
            'participant_id': self.participant_id,
            'quantity': kwargs['quantity'],  # Wh
            'price': kwargs['price'],  # $/kWh
            'source': kwargs['source'],
            'time_delivery': time_delivery
        }
        await self.__client.emit('ask', ask_entry)

    async def ask_success(self, message):
        await self.__ledger.ask_success(message)

    async def bid_success(self, message):
        await self.__ledger.bid_success(message)

    async def settle_success(self, message):
        await self.__ledger.settle_success(message)
        return message['commit_id']

    async def __update_time(self, message):
        # synchronizes time with market
        duration = message['duration']
        start_time = message['time']
        self.__timing.update({
            'timezone': message['timezone'],
            'duration': duration,
            'last_round': tuple(message['last_round']),
            'current_round': tuple(message['current_round']),
            'last_settle': tuple(message['last_settle']),
            'next_settle': tuple(message['next_settle']),
            'stale_round': (start_time - duration * 10, start_time - duration * 9)
        })

    async def start_round(self, message):
        """Sequence of actions during each round
        Currently only for simulation mode.
        Real time mode needs slight modifications.

        Args:
            message ([type]): [description]
        """
        # start of current time step
        await self.__update_time(message)
        self.__market_info.update(message['market_info'])
        await self.__ledger.clear_history(self.__timing['stale_round'])
        self.__market_info.pop(str(self.__timing['stale_round']), None)
        # print(self.__market_info)
        # agent_act tells what actions controller should perform
        # controller should perform those actions accordingly, but will have the option not to
        next_actions = await self.trader.step()
        # next_actions = await self.trader.act()
        await self.__take_actions(next_actions)
        # await self.trader.learn()
        if hasattr(self, 'storage'):
            await self.storage.step()

        # metering energy should happen right at the end of the current round for maximum accuracy
        # in real-time mode there would have to be a timeout function
        # this is currently OK for simulation mode
        await self.__meter_energy(self.__timing['current_round'])
        # await self.__client.emit('end_turn', namespace='/market')
        await self.__client.emit('end_turn')

    async def __read_profile(self, time_interval):
        """Fetches energy profile for one timestamp from database

        Args:
            time_interval ([type]): [description]

        Returns:
            [type]: [description]
        """
        db = self.__profile['db']
        table = self.__profile['db_table']
        # query = table.select().where(table.c.tstamp == time_interval[1])
        query = table.select().where(table.c.time == time_interval[1])
        async with db.transaction():
            row = await db.fetch_one(query)
        return utils.process_profile(row=row,
                                     gen_scale=self.__profile_params['generation_scale'],
                                     load_scale=self.__profile_params['load_scale'])

    # def __process_profile(self, row):
    #     """Processes raw readings fetches from database into generation and consumption in integer Wh.
    #
    #     Also scales if scaling is defined in configuration.
    #     Right now the format is for eGauge. Functionality will be expanded as more data sources are introduced.
    #
    #     Args:
    #         row ([type]): [description]
    #
    #     Returns:
    #         [type]: [description]
    #     """
    #
    #     if row is not None:
    #         consumption = int(round(self.__profile_params['load_scale'] * (row['grid'] + row['solar+']), 0))
    #         generation = int(round(self.__profile_params['generation_scale'] * row['solar+'], 0))
    #         return generation, consumption
    #     return 0, 0

    async def __meter_energy(self, time_interval):
        """Sends submetering data to the Market

        In simulation mode, the data is sent at the end of the current step
        In real-time mode, the data is sent at the beginning of the next step

        Args:
            time_interval ([type]): [description]

        Returns:
            [type]: [description]
        """

        if not self.server_online:
            return False

        if not self.market_connected:
            return False

        self.__meter = await self.__allocate_energy(time_interval)
        await self.__client.emit('meter_data', self.__meter)
        return True

    async def __allocate_energy(self, time_interval):
        """
        This function performs virtual sub metering.
        energy generated is allocated to sources by priority:
        1. settlements
        2. self consumption
        battery, then
        other loads
        3. grid
        """

        self.__timing['last_deliver'] = time_interval
        meter = {
            'time_interval': time_interval,

            # generation is NET from each source
            # self consumed energy are allocated to consumption
            # this makes market code more efficient
            'generation': {
                'solar': 0,
                'bess': 0
            },
            'consumption': {
                # consumption keeps track of self consumption by source
                # other denotes energy flowing in from the outside
                'bess': {
                    'solar': 0  # solar from self
                },
                'other': {
                    'solar': 0,  # solar from self
                    'bess': 0,  # bess from self
                    'external': 0
                }
            }
        }

        # step 1. gather settlements
        settled_solar = 0
        settled_bess = 0
        if time_interval in self.__ledger.settled:
            asks = self.__ledger.settled[time_interval]['asks']

            for ask in asks.values():
                if ask['source'] == 'solar':
                    settled_solar += ask['quantity']
                if ask['source'] == 'bess':
                    settled_bess += ask['quantity']

        # step 2. get battery activity
        bess_charge = 0
        bess_discharge = 0

        if hasattr(self, 'storage'):
            # bess_activity = self.storage.last_activity
            bess_activity = await self.storage.check_schedule(time_interval)
            bess_activity = bess_activity[time_interval]['energy_scheduled']
            bess_charge = bess_activity if bess_activity > 0 else 0
            bess_discharge = -bess_activity if bess_activity < 0 else 0

        # step 3. get readings from meter (profile)
        solar_generation, residual_consumption = await self.__read_profile(time_interval)

        # step 4. allocate energy
        # use solar for settlement
        residual_solar = max(0, solar_generation - settled_solar)
        if residual_solar == 0:
            meter['generation']['solar'] += solar_generation
        else:
            meter['generation']['solar'] += settled_solar

        # use residual solar to charge battery (if battery was charged)
        if residual_solar > 0 and bess_charge > 0:
            if residual_solar <= bess_charge:
                meter['consumption']['bess']['solar'] += residual_solar
                bess_charge -= residual_solar
                residual_solar -= residual_solar
            elif residual_solar > bess_charge:
                meter['consumption']['bess']['solar'] += bess_charge
                residual_solar -= bess_charge
                bess_charge -= bess_charge

        # use residual solar for other loads
        if residual_solar > 0:
            if residual_solar <= residual_consumption:
                meter['consumption']['other']['solar'] += residual_solar
                residual_consumption -= residual_solar
                residual_solar -= residual_solar
            elif residual_solar > residual_consumption:
                meter['consumption']['other']['solar'] += residual_consumption
                residual_solar -= residual_consumption
                residual_consumption -= residual_consumption

        # if battery was discharged
        if bess_discharge > 0:
            if bess_discharge <= residual_consumption:
                meter['consumption']['other']['bess'] += bess_discharge
                residual_consumption -= bess_discharge
                bess_discharge -= bess_discharge
            elif bess_discharge > residual_consumption:
                meter['consumption']['other']['bess'] += residual_consumption
                bess_discharge -= residual_consumption
                residual_consumption -= residual_consumption

        meter['generation']['solar'] += residual_solar
        meter['generation']['bess'] += bess_discharge
        meter['consumption']['other']['external'] += residual_consumption
        meter['consumption']['other']['external'] += bess_charge

        return meter

    async def __take_actions(self, actions):
        """Processes the actions given by the agent

        Args:
            actions ([type]): [description]
        """
        # actions must come in the following format:
        # actions = {
        #     'bess': {
        #         time_interval: scheduled_qty
        #     },
        #     'bids': {
        #         time_interval: {
        #             'quantity': qty,
        #             'price': dollar_per_kWh
        #         }
        #     },
        #     'asks' {
        #         source: {
        #             time_interval: {
        #                 'quantity': qty,
        #                 'price': dollar_per_kWh?
        #             }
        #         }
        #     }
        # }

        # Battery charging or discharging action
        if 'bess' in actions and hasattr(self, 'storage'):
            for time_interval in actions['bess']:
                await self.storage.schedule_energy(actions['bess'][time_interval], ast.literal_eval(time_interval))
        # Bid for energy
        if 'bids' in actions:
            for time_interval in actions['bids']:
                quantity = actions['bids'][time_interval]['quantity']
                price = round(actions['bids'][time_interval]['price'], 4)
                await self.bid(quantity=quantity,
                               price=price,
                               time_delivery=ast.literal_eval(time_interval))
        # Ask to sell energy
        if 'asks' in actions:
            for source in actions['asks']:
                for time_interval in actions['asks'][source]:
                    quantity = actions['asks'][source][time_interval]['quantity']
                    price = round(actions['asks'][source][time_interval]['price'], 4)
                    await self.ask(quantity=quantity,
                                   price=price,
                                   source=source,
                                   time_delivery=ast.literal_eval(time_interval))

    def reset(self):
        self.__ledger.reset()
        self.__extra_transactions.clear()
        self.__market_info.clear()
        self.__meter.clear()
        self.__timing.clear()

    # Keep participant connected to server
    async def ping(self):
        while self.run:
            await self.delay(10)
            if self.server_online:
                await self.__client.emit('ping')
            continue
        await self.__client.sleep(5)
        await self.__client.disconnect()
        os.kill(os.getpid(), signal.SIGINT)
        # raise SystemExit

# import socketio
# class NSMarket(socketio.AsyncClientNamespace):
#     def __init__(self, participant):
#         super().__init__(namespace='/market')
#         self.participant = participant
#
#     # async def on_connect(self):
#     #     await self.participant.join_market()
#
#     # async def on_disconnect(self):
#     #     pass
#
#     async def on_re_register_participant(self, message):
#         await self.participant.join_market()
#
#     async def on_update_market_info(self, market_id):
#         if market_id == self.participant.market_id:
#             self.participant.market_connected = True
#
#     async def on_start_round(self, message):
#         await self.participant.start_round(message)
#
#     async def on_ask_success(self, message):
#         await self.participant.ask_success(message)
#
#     async def on_bid_success(self, message):
#         await self.participant.bid_success(message)
#
#     async def on_settled(self, message):
#         return await self.participant.settle_success(message)
#
#     async def on_return_extra_transactions(self, message):
#         await self.participant.update_extra_transactions(message)
#
# if __name__ == '__main__':
#     pass


import timeit
from copy import copy

import Algorithms.point_cloud_generator as pcg
import Algorithms.alphacomplexwrapper as acw
import Algorithms.random_discrete_morse as rdm
import Algorithms.column_algo.column_algorithm as ca
import Examples.rand_k_n_p as rnp
import Examples.davides_eight_point_delaunay as dd

from MorseAndFiltrations.DMF_to_filtration import DMF_to_filtration
from MorseAndFiltrations.filtration_to_DMF import filtration_to_DMF

import scipy.stats as scs
import numpy as np
import matplotlib.pyplot as plt
import tikzplotlib

def uniform_comparison(n,dim, console_outs = False):
    # points = pcg.generate_n_points(n,dim)

    #
    # if(dim == 2):
    #     points = pcg.multivariate_gaussian(n,[0,0],
    #                                         [[1,0],[0,1]])
    # else:
    #     points = pcg.multivariate_gaussian(n, [0, 0, 0],
    #                                         [[1, 0, 0], [0, 1, 0], [0 ,0 ,1]])
    #

    if(dim == 2):
        points = pcg.gaussian_mixture_model(n,[[0,0],[1,1],[0,3]],
                                            [[[1,0],[0,1]],[[1,0],[0,1]],[[1,0],[0,1]]])
    else:
        points = pcg.gaussian_mixture_model(n, [[0, 0, 0], [1, 1, 1], [0, 3, 0.5]],
                                            [[[1, 0, 0], [0, 1, 0], [0 ,0 ,1]],
                                             [[1, 0, 0], [0, 1, 0], [0 ,0 ,1]],
                                             [[1, 0, 0], [0, 1, 0], [0 ,0 ,1]]])

    alpha = acw.AlphaComplexWrapper(points, False)
    filtration = [elem[0] for elem in alpha.filtration]

    start = timeit.default_timer()
    mat1 = ca.build_boundary_matrix_from_filtration(filtration)
    steps_without,_,_ = ca.column_algorithm(mat1)
    stop = timeit.default_timer()
    without_clearing_time = stop-start
    if(console_outs):
        print("WITHOUT CLEARING TIME: ", stop - start)
        print("WITHOUT CLEARING ADDITIONS: ", steps_without)

        print("--------------------")
        print("LENGTH OF FILTRATION: ", len(filtration))

    start = timeit.default_timer()
    crit, pairs, clear = filtration_to_DMF(filtration)
    if (console_outs):
        print("ELEMENTS IN PAIRS: ", len(pairs) * 2)
        print("--------------------")

    mat2 = ca.build_boundary_matrix_from_filtration_and_clear(filtration,clear)
    steps_apparent,_,_ = ca.column_algorithm(mat2)
    stop = timeit.default_timer()
    apparent_clearing_time = stop-start

    if (console_outs):
        print("APPARENT CLEARING TIME: ", stop - start)
        print("APPARENT CLEARING ADDITIONS: ", steps_apparent)

    s = without_clearing_time/ apparent_clearing_time
    a = 100-(100/steps_without[0] * steps_apparent[0])

    if (console_outs):
        print("Speedupfactor by apparents: ", s)
        print("Additions cleared via apparents : ", a)

    return s,a,len(pairs)*200/len(filtration)

def get_average_speedup_factor(n,runs):
    speed2 = []
    speed3 = []
    speed4 = []

    additions2 = []
    additions3 = []
    additions4 = []

    percentage2 = []
    percentage3 = []
    percentage4 = []

    estimated_time_till_finish = [0]

    for i in range(runs):
        if(i > 0):
            print("Step: ", i, "Estimated time till finish: ", (sum(estimated_time_till_finish)/i)*(runs-i))
        start = timeit.default_timer()
        s2,a2,p2 = uniform_comparison(n,2)
        speed2.append(s2)
        additions2.append(a2)
        percentage2.append(p2)

        s3,a3,p3 = uniform_comparison(n,3)
        speed3.append(s3)
        additions3.append(a3)
        percentage3.append(p3)
        stop = timeit.default_timer()

        estimated_time_till_finish.append(stop-start)

        # s4,a4 = uniform_comparison(n,4)
        # speed4.append(s4)
        # additions4.append(a4)

    fit_data(speed2,"s2")
    fit_data(speed3,"s3")
    # fit_data(speed4,4)

    fit_data(additions2, "a2")
    fit_data(additions3, "a3")

    fit_data(percentage2, "p2")
    fit_data(percentage3, "p3")

def fit_data(data,id):
    mu, std = scs.norm.fit(data)

    # Plot the histogram.
    plt.hist(data, bins=100, alpha=0.6, histtype='bar', density=True)
    # Plot the PDF.
    xmin, xmax = plt.xlim()
    x = np.linspace(xmin, xmax, 1000)
    #p = scs.norm.pdf(x, mu, std)
    #plt.plot(x, p, 'k', linewidth=2)
    title = id + "_mean = " + str(mu)
    plt.title(title)
    tikzplotlib.save("apparent_gains_gaussian_mixture_300_points_value_" + str(id) + "_runs_" + str(len(data)) + ".tex")
    plt.show()

def analyse_gap(n, iterations):
    big_elements = []
    big_percentage = []
    big_steps = []

    small_elements = []
    small_percentage = []
    small_steps = []

    saved_additions = []

    for i in range(iterations):
        print(i)
        points = pcg.multivariate_gaussian(n, [0, 0, 0], [[1, 0, 0], [0, 1, 0], [0 ,0 ,1]])

        alpha = acw.AlphaComplexWrapper(points, False)
        filtration = [elem[0] for elem in alpha.filtration]

        mat1 = ca.build_boundary_matrix_from_filtration(filtration)
        steps_without,_,_ = ca.column_algorithm(mat1)
        crit, pairs, clear = filtration_to_DMF(filtration)

        mat2 = ca.build_boundary_matrix_from_filtration_and_clear(filtration,clear)
        steps_apparent,_,_ = ca.column_algorithm(mat2)

        a = 100-(100/steps_without[0] * steps_apparent[0])
        saved_additions.append(a)

        if(a < 50):
            small_elements.append(len(filtration))
            small_percentage.append((100 * len(pairs)*2)/len(filtration))
            small_steps.append(steps_without[0])
        else:
            big_elements.append(len(filtration))
            big_percentage.append((100 * len(pairs)*2)/len(filtration))
            big_steps.append(steps_without[0])

    print("SMALL_ELEMENTS_MEAN", sum(small_elements)/len(small_elements))
    print("SMALL_PERCENTAGE_MEAN", sum(small_percentage)/len(small_percentage))
    print("SMALL_STEPS_MEAN", sum(small_steps)/len(small_steps))

    print("###############")

    print("BIG_ELEMENTS_MEAN", sum(big_elements) / len(big_elements))
    print("BIG_PERCENTAGE_MEAN", sum(big_percentage) / len(big_percentage))
    print("BIG_STEPS_MEAN", sum(big_steps) / len(big_steps))

    fit_data(saved_additions,"_")

get_average_speedup_factor(300,100)
# analyse_gap(30,100)