# to
# i can webscrape it without having to render it
# i just have to fish out the
# <div class="js-tweet-text-container"> </div>
# objects

# stages
# 1 download a searchpage every 4 seconds
# extract the tweet objects from the page into an array
# create a coherent timeline (elimiate doubles)
# save to disk in some format

import tweepy
import queue
import datetime
import pprint
import time
import multiprocessing
import threading
from main import ROOTDIR, dir_sep as dirsep
from utils import handle_ratelimit, escape_string, log
import csv

raw_tweet_queue = queue.Queue()
downloader_done = False

# TODO make it so that you download a set number of tweets rather
# TODO than a daterange cuz twitter just doesn't want me to have nice thing

def test(api):
    search_term = '@ISPuuuv'
    #search_term = '@Potus'
    dump_dir = ROOTDIR + dirsep + search_term + dirsep
    today = datetime.date.today()
    start_date = today.replace(month=today.month-1)
    end_date = start_date.replace(day=start_date.day +1)

    dlr = Tweet_Downloader()
    writer = Tweets_To_Csv()

    dl_process = threading.Thread(target=dlr.daterange_scrape, args=(api, search_term, start_date, end_date))
    dl_process.start()
    # join ends the thread once it's finished
    dl_process.join()

    writer_process = threading.Thread(target=writer.run, args=(dump_dir, search_term, start_date))
    writer_process.start()
    writer_process.join()

class Tweet_Downloader:
    def daterange_scrape(self, api, search_term, date_start, date_end):
        # the date arguments must be datetime.date objects
        log("starting datrange scrape")

        results = []
        since_id_search_term = self.__date_to_search_term(date_start, date_end, search_term)
        print(api.search(since_id_search_term))
        # this gets the since_id associated with the search term
        while not len(results) > 0:
            log("getting results")
            try:
                for page in tweepy.Cursor(api.search, q=since_id_search_term).pages():
                    results = page
                    print(page[0].created_at)
                    #break
            except Exception as e:
                log(str(e))
                handle_ratelimit("since id getter")

        since_id = results[0].id

        self.__download_tweets(api, search_term, since_id, date_end)

    def __date_to_search_term(self, date_start, date_end, search_term):
        # searchterm since:2017-09-18 until:2017-09-19
        # this string used as searchterm will search the daterange specified
        return str(search_term + " since:" + str(date_start) + " until:" + str(date_end))

    def __tweets_to_queue(self, tweets):
        # if in con't place the items in the queue it waits instead of throwing an exept (apparently)
        for tweet in tweets:
            global raw_tweet_queue
            raw_tweet_queue.put(tweet)

    def __check_stream_end(self, results, stream_end):
        # stream_end is a datetime.datetime object of where the stream ends.
        #stream_end = datetime(stream_end.year, stream_end.month, stream_end.day)

        if results[0].created_at > datetime.datetime(stream_end.year, stream_end.month, stream_end.day):
            return True
        return False

    def __truncate_stream_end(self, page, stream_end_date):
        # cuts of the tweets that fall outside of the time range
        return page#TODO

    def __download_tweets(self, api, search_term, since_id, stream_end_date):

        end_of_stream = False
        i = 1
        while not end_of_stream:
            print("trying page download")
            try:
                for page in tweepy.Cursor(api.search, q=search_term, since_id=since_id).pages():
                    print("downloaded page")
                    # page is a list of statuses

                    if self.__check_stream_end(page, stream_end_date):
                        end_of_stream = True

                        page = self.__truncate_stream_end(page, stream_end_date)
                        self.__tweets_to_queue(page)  # adds truncated page onto queue before exiting
                        break  # breaks out of try exept loop
                        # if break is not here it will continue until it hits the rate limit

                    #if i % 5 == 0:
                    log("got page " + str(i))
                    i += 1

                    self.__tweets_to_queue(page)

            except Exception as e:
                log(str(e))
                handle_ratelimit("dler")
        global downloader_done
        downloader_done = True

# methods for saving tweet data to a csv file
class Tweets_To_Csv:
    def run(self, dump_dir, searchterm, date):
        filename = searchterm + " " + str(date) + ".csv"
        with open(filename, 'w', newline='', encoding='utf-8') as csvfile:
            fieldnames = ["posted_at", "tweet_id", "in_reply_to_tweet_id", "in_reply_to_user_id", "user_id",
                          "user_name", "user_screen_name", "user_verified", "tweet_text", "retweets_no", "likes",
                          "application_used", "user_location", "coordinates", "user_timezone",
                          "follower_count", "following_count", "user_likes_no", "lang", "user_created"]

            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
            writer.writeheader()
            # while not downloader_done:
            i = 1

            while True:
                q = raw_tweet_queue
                if q.empty():
                    log("waiting 3 seconds for tweets to download")
                    time.sleep(3)
                else:
                    self.__mask_and_write(q.get(), writer)
                    if i % 30 == 0:
                        log(str(i) + " tweets encoded")
                    i+=1

                if q.empty() and downloader_done:
                    break

    def __mask_and_write(self, tweet, writer):
        posted_at = str(tweet.created_at)  # have to cast cause it's a datetime object
        tweet_id = tweet.id_str
        in_reply_to_tweet_id = tweet.in_reply_to_status_id_str
        in_reply_to_user_id = tweet.in_reply_to_user_id_str
        user_id = tweet.user.id_str
        user_name = tweet.user.name
        user_screen_name = tweet.user.screen_name
        user_verified = 0
        if tweet.user.verified:  # this is to save space
            user_verified = 1
        tweet_text = tweet.text
        tweet_text = escape_string(tweet_text)
        # replies_no = tweet.? #don't think it's in here
        retweets_no = tweet.retweet_count
        likes = tweet.favorite_count
        application_used = tweet.source_url
        # possibly_sensitive = 0
        # if tweet.retweeted_status.possibly_sensitive:
        #    possibly_sensitive = 1
        user_location = tweet.user.location
        coordinates = tweet.coordinates
        user_timezone = tweet.user.time_zone
        follower_count = tweet.user.followers_count
        following_count = tweet.user.friends_count
        user_likes_no = tweet.user.favourites_count
        lang = tweet.lang
        user_created = str(tweet.user.created_at)  # have to cast cause it's a datetime object

        writer.writerow({"posted_at": posted_at, "tweet_id": tweet_id, "in_reply_to_tweet_id": in_reply_to_tweet_id,
                     "in_reply_to_user_id": in_reply_to_user_id, "user_id": user_id, "user_name": user_name,
                     "user_screen_name": user_screen_name, "user_verified": user_verified, "tweet_text": tweet_text,
                     "retweets_no": retweets_no, "likes": likes, "application_used": application_used, "user_location":
                         user_location, "coordinates": coordinates, "user_timezone": user_timezone, "follower_count":
                         follower_count, "following_count": following_count, "user_likes_no": user_likes_no, "lang": lang,
                     "user_created": user_created})


from os import get_inheritable
from kollider_api_client.ws import KolliderWsClient
from kollider_api_client.rest import KolliderRestClient
from utils import *
from console_logger import *
from lnd_client import LndClient
from kollider_msgs import OpenOrder, Position, TradableSymbol, Ticker
from time import sleep
from threading import Lock
import json
from math import floor
import uuid
from pprint import pprint
import threading

import zmq

SATOSHI_MULTIPLIER = 100000000

SOCKET_ADDRESS = "tcp://*:5556"

class HedgerState(object):
	position_quantity = 0
	ask_open_order_quantity = 0
	bid_open_order_quantity = 0
	target_quantity = 0
	target_value = 0
	is_locking = False
	lock_price = None
	side = None
	predicted_funding_payment = 0

	def to_dict(self):
		return {
			"position_quantity": self.position_quantity,
			"bid_open_order_quantity": self.bid_open_order_quantity,
			"ask_open_order_quantity": self.ask_open_order_quantity,
			"target_quantity": self.target_quantity,
			"target_value": self.target_value,
			"is_locking": self.is_locking,
			"lock_price": self.lock_price,
			"side": self.side,
			"predicted_funding_payment": self.predicted_funding_payment
		}

class Wallet(object):

	def __init__(self):
		self.channel_balance = 0
		self.nchain_balance = 0
		self.kollider_balance = 0

	def update_channel_balance(self, balance):
		self.channel_balance = balance

	def update_onchain_balance(self, balance):
		self.onchain_balance = balance

	def update_kollider_balance(self, balance):
		self.kollider_balance = balance

	def to_dict(self):
		return {
			"channel_balance": self.channel_balance,
			"onchain_balance": self.onchain_balance,
			"kollider_balance": self.kollider_balance
		}

	def total_ln_balance(self):
		return self.channel_balance + self.kollider_balance

class HedgerEngine(KolliderWsClient):
	def __init__(self, lnd_client):
		# Orders that are currently open on the Kollider platform.
		self.open_orders = {}
		# Positions that are currently open on the Kollider platform.
		self.positions = {}
		self.current_index_price = 0
		self.current_mark_price = 0
		self.target_fiat_currency = "USD"
		self.target_symbol = "BTCUSD.PERP"
		self.target_index_symbol = ".BTCUSD"
		self.ws_is_open = False
		self.contracts = {}

		self.hedge_value = 0
		self.hedge_proportion = 0
		self.hedge_side = None
		self.target_leverage = 100

		# Last hedge state.
		self.last_state = None

		# Summary of the connected node.
		self.node_info = None

		# Order type that is used to make trades on Kollider.
		self.order_type = "Market"

		self.wallet = Wallet()

		self.lnd_client = lnd_client

		self.last_ticker = Ticker()

		self.received_tradable_symbols = False

		# Average hourly funding rates for each symbol. Used
		# as an prediction of the next funding rate.
		self.average_hourly_funding_rates = {}

		self.is_locked = True

	def to_dict(self):
		average_funding = self.average_hourly_funding_rates.get(self.target_symbol)
		average_funding = average_funding if average_funding else 0
		return {
			"node_info": self.node_info,
			"wallet": self.wallet.to_dict(),
			"current_index_price": self.current_index_price,
			"current_mark_price": self.current_mark_price,
			"last_state": self.last_state.to_dict(),
			"target_fiat_currency": self.target_fiat_currency,
			"target_symbol": self.target_index_symbol,
			"target_index_symbol": self.target_index_symbol,
			"hedge_value": self.hedge_value,
			"hedge_proportion": self.hedge_proportion,
			"average_hourly_funding_rate": average_funding
		}

	def set_params(self, **args):
		self.hedge_proportion = args.get("hedge_proportion") if args.get("hedge_proportion") else 0
		self.hedge_side = args.get("hedge_side") if args.get("hedge_side") else None
		self.target_fiat_currency = args.get("target_fiat_currency") if args.get("target_fiat_currency") else None
		self.target_symbol = args.get("target_symbol") if args.get("target_symbol") else None
		self.target_index_price = args.get("target_index_symbol") if args.get("target_index_symbol") else None
		self.target_leverage = args.get("target_leverage") if args.get("target_leverage") else None
		self.order_type = args.get("order_type") if args.get("order_type") else "Market"

	def on_open(self, event):
		self.auth()
		sleep(1)
		self.sub_index_price([self.target_index_symbol])
		self.sub_mark_price([self.target_symbol])
		self.sub_ticker([self.target_symbol])
		self.sub_position_states()
		self.fetch_positions()
		self.fetch_open_orders()
		self.fetch_tradable_symbols()
		self.fetch_ticker(self.target_symbol)
		self.fetch_balances()
		self.ws_is_open = True

	def on_pong(self, ctx, event):
		None

	def on_error(self, ctx, event):
		pass

	def on_message(self, _ctx, msg):
		msg = json.loads(msg)
		t = msg["type"]
		data = msg["data"]
		if t == 'authenticate':
			if data["message"] == "success":
				self.is_authenticated = True
			else:
				print("Auth Unsuccessful: {}".format(data))
				self.is_authenticated = False
				self.__reset()

		elif t == 'index_values':
			self.current_index_price = float(data["value"])

		elif t == 'mark_prices':
			self.current_mark_price = float(data["price"])

		elif t == 'positions':
			print("Received positions.")
			positions = data["positions"]
			for key, value in positions.items():
				self.positions[key] = Position(msg=value)

		elif t == 'open_orders':
			print("Received open orders")
			open_orders = data["open_orders"]
			for symbol, open_orders in open_orders.items():
				for open_order in open_orders:
					if self.open_orders.get(symbol) is None:
						self.open_orders[symbol] = []
					oo = OpenOrder(msg=open_order)
					self.open_orders[symbol].append(oo)

		elif t == 'tradable_symbols':
			for symbol, contract in data["symbols"].items():
				self.contracts[symbol] = TradableSymbol(msg=contract)
			self.received_tradable_symbols = True

		elif t == 'open':
			open_order = data
			contract = self.get_contract()
			dp = contract.price_dp
			open_order_parsed = OpenOrder(data, dp)
			if self.open_orders.get(open_order_parsed.symbol) is None:
				self.open_orders[open_order_parsed.symbol] = []
			self.open_orders[open_order_parsed.symbol].append(open_order_parsed)

		elif t == 'done':
			symbol = data["symbol"]
			order_id = data["order_id"]
			self.open_orders[symbol] = [open_order for open_order in self.open_orders[symbol] if open_order.order_id != order_id]

		elif t == 'fill':
			symbol = msg["symbol"]
			order_id = msg["order_id"]
			quantity = msg["quantity"]
			orders = self.open_orders.get(symbol)
			if orders is None:
				return
			for order in orders:
				if order.order_id == order_id:
					order.quantity -= quantity

		elif t == 'position_states':
			position = Position(msg=data)
			if position.symbol == self.target_symbol:
				self.positions[self.target_symbol] = position

		elif t == 'ticker':
			self.last_ticker = Ticker(msg=data)

		elif t == 'order_invoice':
			print("Received Pay to Trade invoice for: {}".format(data["margin"]))
			res = self.lnd_client.send_payment(data["invoice"])

		elif t == 'settlement_request':
			print("Received settlement Request")
			amount = data["amount"]
			self.make_withdrawal(amount, "Kollider Trade Settlement")

		elif t == 'balances':
			total_balance = 0
			cash_balance = float(data["cash"])
			if cash_balance > 1:
				self.make_withdrawal(cash_balance, "Kollider Payout")
			total_balance += cash_balance
			isolated_margin = data["isolated_margin"].get(self.target_symbol)
			if isolated_margin is not None:
				total_balance += float(isolated_margin)

			order_margin = data["order_margin"].get(self.target_symbol)
			if order_margin is not None:
				total_balance += float(order_margin)
			self.wallet.update_kollider_balance(total_balance)

		elif t == 'error':
			print("ERROR")
			print(data)

	def make_withdrawal(self, amount, message):
		amt = int(amount)
		res = self.lnd_client.add_invoice(amt, message)
		withdrawal_request = {
			"withdrawal_request": {
				"Ln": {
					"payment_request": res.payment_request,
					"amount": amt,
				}
			}
		}
		self.withdrawal_request(withdrawal_request)

	def cancel_all_orders_on_side(self, side):
		orders = self.open_orders.get(self.target_symbol)

		if orders is None:
			return

		for order in orders:
			if order.side == side:
				self.cancel_order({"order_id": order.order_id, "symbol": self.target_symbol, "settlement_type": "Delayed"})

	def calc_contract_value(self):
		if self.contracts.get(self.target_symbol) is not None:
			contract = self.get_contract()
			price = self.current_mark_price
			if contract.is_inverse_priced:
				return contract.contract_size / price * SATOSHI_MULTIPLIER
			else:
				return contract.contract_size * price * SATOSHI_MULTIPLIER
		else:
			raise Exception("Target contract not available")

	def calc_number_of_contracts_required(self, value_target):
		try:
			value_per_contract = self.calc_contract_value()
			qty_of_contracts = floor(value_target / value_per_contract)
			return qty_of_contracts
		except Exception as e:
			print(e)

	def get_open_orders(self):
		return self.open_orders.get(self.target_symbol)

	def get_open_position(self):
		return self.positions.get(self.target_symbol)

	def get_contract(self):
		return self.contracts.get(self.target_symbol)

	def get_best_price(self, side):
		if side == "Bid":
			return self.last_ticker.best_bid
		else:
			return self.last_ticker.best_ask

	def update_average_funding_rates(self):
		rest_client = KolliderRestClient("http://api.staging.kollider.internal/v1/")
		average_funding_rates = rest_client.get_average_funding_rates()
		for funding_rate in average_funding_rates["data"]:
			self.average_hourly_funding_rates[funding_rate["symbol"]] = funding_rate["mean_funding_rate"]

	def build_target_state(self):
		state = HedgerState()

		target_value = self.wallet.total_ln_balance()
		hedge_value = (target_value * self.hedge_proportion)

		# The number a contracts we need to cover the value.
		target_number_of_contracts = self.calc_number_of_contracts_required(hedge_value)

		# Getting current open orders.
		open_orders = self.get_open_orders()

		open_ask_order_quantity = 0
		open_bid_order_quantity = 0

		current_position_quantity = 0

		if open_orders is not None:
			open_bid_order_quantity = sum([open_order.quantity for open_order in open_orders if open_order.side == "Bid"])
			open_ask_order_quantity = sum([open_order.quantity for open_order in open_orders if open_order.side == "Ask"])

		# Getting current position on target symbol.
		open_position = self.get_open_position()

		if open_position is not None:
			current_position_quantity = open_position.quantity
			state.side = open_position.side
			# If current position is on the wrong side we adding a minus to reflect that.
			if open_position.side != opposite_side(self.hedge_side):
				current_position_quantity = current_position_quantity * -1
			else:
				state.lock_price = open_position.entry_price

		state.target_quantity = target_number_of_contracts
		state.target_value = hedge_value
		state.bid_open_order_quantity = open_bid_order_quantity
		state.ask_open_order_quantity = open_ask_order_quantity
		state.position_quantity = current_position_quantity

		hourly_funding_rate = self.average_hourly_funding_rates.get(self.target_symbol)
		if hourly_funding_rate is not None:
			state.predicted_funding_payment = hedge_value * hourly_funding_rate
		else:
			state.predicted_funding_payment = 0

		self.last_state = state

		return state

	def converge_state(self, state):
		# Nothing needs to be done if target is current.

		target_side = opposite_side(self.hedge_side)

		current = 0
		if target_side == "Bid":
			current = state.bid_open_order_quantity + state.position_quantity
		else:
			current = state.ask_open_order_quantity + state.position_quantity

		target = state.target_quantity

		if target == current:
			self.cancel_all_orders_on_side(self.hedge_side)
			return

		contract = self.get_contract()

		dp = contract.price_dp

		side = None

		if target > current:
			side = opposite_side(self.hedge_side)
			self.cancel_all_orders_on_side(self.hedge_side)

		elif target < current:
			side = self.hedge_side
			if target_side == "Bid" and state.bid_open_order_quantity > 0:
				self.cancel_all_orders_on_side(target_side)
				return
			elif target_side == "Ask" and state.ask_open_order_quantity > 0:
				self.cancel_all_orders_on_side(target_side)
				return

		price = self.get_best_price(side)

		# Adding the order to the top of the book by adding/subtracting one tick.
		if side == "Bid":
			price += contract.tick_size
		else:
			price -= contract.tick_size

		price = int(price * (10**dp))

		qty_required = abs(target - current)

		order = {
			'symbol': self.target_symbol,
			'side': side,
			'quantity': qty_required,
			'leverage': self.target_leverage,
			'price': price,
			'order_type': self.order_type,
			'margin_type': 'Isolated',
			'settlement_type': 'Instant',
			'ext_order_id': str(uuid.uuid4()),
		}

		self.place_order(order)

	def check_state(self, state):
		side = opposite_side(self.hedge_side)
		# If our current position has the right quantity and the right side we are fully locked
		if state.target_quantity == state.position_quantity and state.side == side:
			state.is_locking = False
			self.is_locked = True
		else:
			state.is_locking = True
			self.is_locked = False
		return state

	def print_state(self, state):
		pprint(state.to_dict())
		pprint(self.wallet.to_dict())

	def update_node_info(self):
		try:
			node_info = self.lnd_client.get_info()
			self.node_info = {
				"alias": node_info.alias,
				"identity_pubkey": node_info.identity_pubkey,
				"num_active_channels": node_info.num_active_channels,
			}
		except Exception as e:
			print(e)

	def update_wallet_data(self):
		channel_balances = self.lnd_client.get_channel_balances()
		onchain_balances = self.lnd_client.get_onchain_balance()
		self.wallet.update_channel_balance(channel_balances.balance)
		self.wallet.update_onchain_balance(onchain_balances.total_balance)

	def cli_listener(self):
		context = zmq.Context()
		socket = context.socket(zmq.REP)
		socket.bind(SOCKET_ADDRESS)
		while True:
			message = socket.recv_json()
			if message.get("action") is not None:
				action = message.get("action")
				value = message.get("value")
				if action == "set_hedge_proportion":
					print("here")
					self.hedge_proportion = value
			sleep(0.5)
			msg = self.to_dict()
			socket.send_json(msg)
			sleep(0.5)

	def start(self, settings):
		pprint(settings)
		print("Connecting to Kollider websockets..")
		while not self.ws_is_open:
			pass
		print("Connected to websockets.")

		cycle_speed = settings["cycle_speed"]

		self.set_params(**settings)

		self.update_node_info()

		cli_listener = threading.Thread(target=self.cli_listener, daemon=False)
		cli_listener.start()

		while True:
			# Don't do anything if we haven't received the contracts.
			if not self.received_tradable_symbols and not self.is_authenticated:
				continue

			# Don't do anything if we haven't received mark or index price.
			if self.current_index_price == 0 or self.current_mark_price == 0:
				continue

			# Don't do anything if we have no ticker price.
			if self.last_ticker.last_side is None:
				continue

			self.update_wallet_data()

			self.update_average_funding_rates()

			# Getting current state.
			state = self.build_target_state()
			self.check_state(state)
			# Printing the state.
			# Converging to that state.
			self.converge_state(state)

			sleep(cycle_speed)

if __name__ in "__main__":

	settings = None
	with open('config.json') as a:
		settings = json.load(a)

	kollider_api_key = settings["kollider"]["api_key"]
	kollider_url = settings["kollider"]["ws_url"]

	node_url = settings["lnd"]["node_url"]
	macaroon_path = settings["lnd"]["admin_macaroon_path"]
	tls_path = settings["lnd"]["tls_path"]

	lnd_client = LndClient(node_url, macaroon_path, tls_path)

	rn_engine = HedgerEngine(lnd_client)

	lock = Lock()

	rn_engine.connect(settings["kollider"]["ws_url"], settings["kollider"]["api_key"], settings["kollider"]["api_secret"], settings["kollider"]["api_passphrase"])

	rn_engine.start(settings)


'''
    File: simple_autoencoder.py
    Author: Adam Good
    Description: An example of an autoencoder on the MNIST dataset
'''
import time
import argparse
import os
import torch
from torch import nn
from torch.nn import functional as F
import torch.optim as optim

from data_managment import get_dataloaders
from autoencoder_utils import train_model, print_epoch

SAE_LAYERS = [28*28, 512, 128, 32]

class SimpleAutoEncoder(nn.Module):
    '''
        A basic autoencoder
        input -> hidden -> bottleneck -> hidden -> output
    '''
    def __init__(self):
        super(SimpleAutoEncoder, self).__init__()
        self.input_size = SAE_LAYERS[0]
        self.bottleneck_size = SAE_LAYERS[-1]

        left = SAE_LAYERS[:-1]
        right = SAE_LAYERS[1:]
        encoding_layers = [nn.Linear(l, r) for l, r in zip(left, right)]
        self.bottleneck_layer = encoding_layers.pop()
        self.encoding_layers = nn.ModuleList(encoding_layers)

        left, right = right[::-1], left[::-1]
        decoding_layers = [nn.Linear(l, r) for l, r in zip(left, right)]
        self.output_layer = decoding_layers.pop()
        self.decoding_layers = nn.ModuleList(decoding_layers)

    def encode(self, original):
        '''
            Encode x to bottleneck
        '''
        encoding = original
        # encoding = original.view(-1, 1, self.input_size)
        for layer in self.encoding_layers:
            encoding = F.relu(layer(encoding))
        encoding = self.bottleneck_layer(encoding)
        return encoding

    def decode(self, encoding):
        '''
            Decode x back into the original data
        '''
        reconstruction = encoding
        for layer in self.decoding_layers:
            reconstruction = F.elu(layer(reconstruction))
        reconstruction = torch.sigmoid(self.output_layer(reconstruction))
        # reconstruction = reconstruction.view(-1, 1, 28, 28)
        return reconstruction

    def forward(self, x, *argv):
        '''
            Encode and reconstruct the input x
        '''
        return self.decode(self.encode(x))

    def run_epoch(self, optimizer, loss_fn, data_loader, epoch, log_interval=1, training=True):
        if training:
            self.train()
            epoch_type = "Training"
        else:
            self.eval()
            epoch_type = "Validation"

        dataset_size = len(data_loader.dataset)
        epoch_loss = 0.0
        num_items = 0
        batch_idx = 0
        start = time.time()

        for batch_idx, (data, _) in enumerate(data_loader):
            data = data.view(-1, 1, self.input_size)
            if training:
                optimizer.zero_grad()
            output = self(data)
            loss = loss_fn(data, output)
            epoch_loss += loss.item()
            num_items += len(data)
            if training:
                loss.backward()
                optimizer.step()

            if batch_idx % log_interval == 0:
                elapsed_time = time.time() - start
                print_epoch(epoch, epoch_loss, num_items, dataset_size, elapsed_time, epoch_type)

        elapsed_time = time.time() - start
        print_epoch(epoch, epoch_loss, num_items, dataset_size, elapsed_time, epoch_type, end='\n')

    def save(self, path):
        torch.save(self.state_dict(), path)

    def load(self, path):
        self.load_state_dict(torch.load(path))

def main():
    parser = argparse.ArgumentParser(description="Configure Training of Auto Encoder")
    parser.add_argument('-e', '--epochs', type=int, help='Number of Epochs for Training')
    parser.add_argument('-bs', '--batchsize', type=int, help='Training Batch Size')
    parser.add_argument('-lr', '--learningrate', type=float, default=1e-3, help='Learning Rate')
    parser.add_argument('-v', '--verify', action='store_true', help='Run Verification Epochs')
    parser.add_argument('-p', '--trainingpath', type=str, default='./models', help='Directory for Saving State During Training')
    parser.add_argument('-o', '--output', type=str, help='Name for Output File(s)')
    parser.add_argument('-C', '--clean', action='store_true', help='Cleans out Training Data')
    args = parser.parse_args()

    batch_size = args.batchsize
    num_epochs = args.epochs
    path = args.trainingpath
    output = args.output

    if not os.path.exists(path):
        os.makedirs(path)

    train_loader, test_loader = get_dataloaders(batch_size, batch_size)
    autoencoder = SimpleAutoEncoder()
    optimizer = optim.Adam(autoencoder.parameters(), lr=args.learningrate)

    # loss_fn = lambda x,y: F.binary_cross_entropy(y, x, reduction='sum')
    loss_fn = F.mse_loss
    train_model(autoencoder, optimizer, loss_fn, num_epochs, train_loader, test_loader, verify=args.verify, trainingpath=path, name=output)
    if args.clean:
        from shutil import rmtree
        rmtree(path)

if __name__ == '__main__':
    main()


from django import forms
from django.forms import ModelForm
from django.utils.translation import gettext_lazy as _

from Order.models import Order
from User.models import EndUser
from Product.models import Products

class OrderCreateForm(ModelForm):
    payment= forms.TextInput(attrs={'hidden': True, })
    payment_type= forms.TextInput(attrs={'hidden': True, })

    class Meta:
        model = Order
        fields = ['orderer', 'orderdate',  'cost', 'goods', 'order_status', 'up1', 'up1_sc', 'up2', 'up2_sc']

        widgets = {
                #'orderer':forms.Select(choices=EndUser.objects.all()),
                #'orderdate': _('SelectDateWidget'),
                #'goods': forms.SelectMultiple(choices = Products.objects.all()),
                'orderdate':forms.DateInput(attrs={'type':'date'}),
            }
        labels = {
               # 'username':   _('用户名'),
            }
        help_texts = {
            # 'username':   _('可以用中文'),
            }
        field_classes = {
            #'orderdate':  _('cxCalender'),
            }


from PIL.Image import Image
import torch

import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import Dataset, DataLoader
from torch.optim.lr_scheduler import ReduceLROnPlateau
from tqdm import tqdm
import matplotlib.pyplot as plt
import random
import logging
import json
from pprint import pprint
from math import inf
from sklearn.metrics import accuracy_score
from torchinfo import summary
import pdb
import os
import sys
import time
from datetime import timedelta

sys.path.append("..")

from models.multi_image_vqa import MultiImageVQA
from models.custom_logging import TqdmLoggingHandler
from utils.dataset import MultiImageVQADataset, arrange_batch, VQAOne

import warnings
warnings.filterwarnings('ignore')

torch.manual_seed(42)
start_time = time.time()

# ========================================= < Label Smoothing > =======================================
def linear_combination(x, y, epsilon):
    return epsilon*x + (1-epsilon)*y

def reduce_loss(loss, reduction='mean'):
    return loss.mean() if reduction=='mean' else loss.sum() if reduction=='sum' else loss

#  Implementation of Label smoothing with CrossEntropy and ignore_index
class LabelSmoothingCrossEntropy(nn.Module):
    def __init__(self, epsilon:float=0.1, reduction='mean',ignore_index=-100):
        super().__init__()
        self.epsilon = epsilon
        self.reduction = reduction
        self.ignore_index = ignore_index
    def forward(self, preds, target):
        n = preds.size()[-1]
        log_preds = F.log_softmax(preds, dim=-1)
        loss = reduce_loss(-log_preds.sum(dim=-1), self.reduction)
        nll = F.nll_loss(log_preds, target, reduction=self.reduction,ignore_index=self.ignore_index)
        return linear_combination(loss/n, nll, self.epsilon)

# ========================================= <\ Label Smoothing > =======================================

# ========================================= < Logging > ===========================
log = logging.getLogger(__name__)
log.setLevel(logging.INFO)
log.addHandler(TqdmLoggingHandler())

# ========================================= </ Logging > ==========================
num_epochs = 180
vocab_size = 30000 # from bert
seq_len = 12 # from dataset
feat_dim = 640 # from paper, the final vector vq, vi
embed_size = 768 # from paper, dimention of embedding of each word
n_attention_stacks = 2
hidden_dim_img = feat_dim
batch_size = 20
num_workers = 20
lambda_ = 5
device = 'cuda' if torch.cuda.is_available() else 'cpu'

answers = ['6', 'yes', 'small', 'blue', 'brown', 'red', 'cyan', '1', '0', 'rubber', '7', 'no', 'purple', 'large', '5', 'green', '10', 'cube', '4', '9', '3', 'cylinder', 'metal', '2', 'gray', 'sphere', '8', 'yellow']

class Multi(Dataset):
    def __init__(self):
        self.n_images = 3

    def __len__(self):
        return 20

    def __getitem__(self, idx):
        dct = {
            f'images': [torch.randint(255, size=(3, 448, 448)) / 255 for x in range(1, self.n_images+1)]
        }

        batch_size, seq_len = 1, 5
        dct['ques'] = torch.tensor([500 for _ in range(seq_len)])  # torch.randint(1000, size=(seq_len,))
        dct['ans'] = torch.tensor([30]) # torch.randint(9000-2, size=(1,))
        dct['true_img_index'] = 1 # random.randint(0, self.n_images-1)
        return dct

ds = MultiImageVQADataset('/home/prabhu/vqa/models/cleaned.json', '/home/prabhu/CLEVR_v1.0/images/val', '/home/prabhu/Tiny/tiny-imagenet-200/test/images')
# ds = VQAOne("/home/prabhu/VQA1.0/v2_mscoco_train2014_annotations_cleaned.json", "/home/prabhu/VQA1.0/v2_OpenEnded_mscoco_train2014_questions.json", "/home/prabhu/VQA1.0/train2014")
datasetLength = len(ds)

# answer_tokens = ds.tokenizer.tokenize(answers)
# answer_indices = ds.tokenizer.convert_tokens_to_ids(answer_tokens)
# crossentropy_weights = [0 for _ in range(30000)]
# for answer_index in answer_indices:
#     crossentropy_weights[answer_index] = 1 / len(answer_indices)

print("Dataset Loaded. ", datasetLength)
# pdb.set_trace()
train_, validation_ = torch.utils.data.random_split(ds, [int(0.8*datasetLength), len(ds) - int(0.8*datasetLength)], generator=torch.Generator().manual_seed(42))
dl_train = DataLoader(train_, batch_size=batch_size, num_workers=num_workers, collate_fn=arrange_batch, shuffle=True)
dl_val = DataLoader(validation_, batch_size=batch_size+10, num_workers=num_workers, collate_fn=arrange_batch, shuffle=True)

model = MultiImageVQA(feat_dim, vocab_size, embed_size, n_attention_stacks, hidden_dim_img)
state_dict = torch.load('/home/prabhu/vqa/models/exp89/models/exp0/weights.pth')
# # print(*state_dict.keys(), sep='\n')
# del state_dict['pred.4.weight']
# del state_dict['pred.4.bias']
model.load_state_dict(state_dict['state_dict'], strict=False)
print(f'Loaded model weights....')
model = model.to(device)
# model = nn.DataParallel(model)

criterian = nn.CrossEntropyLoss() # LabelSmoothingCrossEntropy() #
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
scheduler = ReduceLROnPlateau(optimizer, verbose=True, patience=6, min_lr=1e-6, factor=0.5)

# ============================== < Set up the project dir > =============================

PROJECT_DIR = '/home/prabhu/vqa/models'
counts = list(filter(lambda x: x.startswith("exp"), os.listdir(PROJECT_DIR)))
# print(counts)
counts = list(map(lambda x: int(x.replace('exp', '')), counts))

EXP_DIR = f'exp{max(counts) + 1}'

os.mkdir(os.path.join(PROJECT_DIR, EXP_DIR))
# ============================== <\ Set up the project dir > =============================

# batch = next(dl)
# for name, param in model.named_parameters():
#     if param.requires_grad:
#         print(name, end=' ')

# for inx, x in enumerate(dl_train):
#     batch = x
#     break
# summary(model, input_data=(batch, batch['ques_features']))
# exit(0)

# dl_train, dl_val = dl, dl

IMG_CLS_CONST, IMG_CLS_FACTOR = 600, 0.5

epoch, train_losses, val_losses, best_val_loss = 0, [], [], inf
word_accuracy, image_accuracy = {'train': [], 'val': []}, {'train': [], 'val': []}
losses = {
    'train': {
        'image_loss': [],
        'word_loss': []
    },
    'val': {
        'image_loss': [],
        'word_loss': []
    }
}
for epoch in tqdm(range(num_epochs), desc=f"on epoch {epoch}"):

    for dataloader, phase in [(dl_train, 'train'),(dl_val, 'val')]:
        if phase == 'train':
            model.train()
        elif phase == 'val':
            model.eval()

        phase_loss, phase_word_loss, phase_image_loss = 0, 0, 0
        phase_word_accuracy, phase_image_accuracy = 0, 0
        for batch in tqdm(dataloader, desc=f"iter {phase} | time: {timedelta(seconds=time.time() - start_time)}"):
            # batch = x
            optimizer.zero_grad()
            keys = batch.keys()

            for key in keys:
                batch[key] = batch[key].to(device)

            if phase == 'val':
                with torch.no_grad():
                    attention_weights, out = model(batch, batch['ques_features'])
            else:
                attention_weights, out = model(batch, batch['ques_features'])

            # log.info(attention_weights)
            pred = torch.argmax(out, dim=1)
            tqdm.write(f'pred: {pred.clone().detach().cpu().tolist()} {torch.argmax(attention_weights, dim=1).clone().detach().cpu().tolist()}\nAns:  {batch["ans"].squeeze(-1).clone().detach().cpu().tolist()} {batch["true_img_index"].squeeze(-1).clone().detach().cpu().tolist()}')
            # tqdm.write(f'Debug: {attention_weights.shape}, {batch["true_img_index"]}')
            # print('out: ', out.shape, 'ans: ', batch['ans'].shape, batch['ans'].detach().cpu().tolist())
            word_loss = criterian(out, batch['ans'].squeeze(-1))
            # torch.tensor([0]).to(device) #
            img_classification_loss = criterian(attention_weights, batch['true_img_index'].squeeze(-1))
            #tqdm.write(f'wetmux ight: {attention_weights}')
            #continue
            loss = (lambda_ * word_loss) # + (IMG_CLS_CONST * img_classification_loss)
            phase_word_loss += lambda_ * word_loss.item() * len(batch['ans'])
            phase_image_loss += img_classification_loss.item() * len(batch['ans'])
            phase_loss += loss.item() * len(batch['ans'])

            phase_word_accuracy += accuracy_score(
                batch['ans'].squeeze(-1).clone().detach().cpu().tolist(),
                pred.clone().detach().cpu().tolist(),
                normalize=False
                )

            phase_image_accuracy += accuracy_score(
                batch["true_img_index"].squeeze(-1).clone().detach().cpu().tolist(),
                torch.argmax(attention_weights, dim=1).clone().detach().cpu().tolist(),
                normalize=False
                )

            if phase == 'train':
                # train_losses.append(loss.detach().cpu().item())
                loss.backward()
                optimizer.step()

            elif phase == 'val':
                pass
                # val_losses.append()
                # scheduler.step(loss)

        tqdm.write(f'phase: {phase} | loss: {phase_loss / len(dataloader.dataset):.3f}, word_Loss: {phase_word_loss / len(dataloader.dataset):.3f}, img_cls_loss: {phase_image_loss / len(dataloader.dataset):.3f}, img_cls_cnst: {IMG_CLS_CONST} | word_accuracy: {phase_word_accuracy / len(dataloader.dataset):.3f} | LR: {optimizer.param_groups[0]["lr"]}')

        word_accuracy[phase].append(phase_word_accuracy / len(dataloader.dataset))
        image_accuracy[phase].append(phase_image_accuracy / len(dataloader.dataset))

        losses[phase]['word_loss'].append(phase_word_loss / len(dataloader.dataset))
        losses[phase]['image_loss'].append(phase_image_loss / len(dataloader.dataset))

        if phase == 'train':
            train_losses.append(phase_loss / len(dataloader.dataset))
        else:
            val_losses.append(phase_loss / len(dataloader.dataset))
            if best_val_loss > phase_loss / len(dataloader.dataset):
                best_val_loss = phase_loss / len(dataloader.dataset)
                torch.save({'state_dict': model.state_dict(), 'lr': optimizer.param_groups[0]["lr"]}, os.path.join(EXP_DIR, 'weights.pth'))
            scheduler.step(phase_loss)
            tqdm.write(f'Val Loss: {best_val_loss:.3f}, Saving weights....')

    IMG_CLS_CONST = IMG_CLS_CONST * IMG_CLS_FACTOR
    IMG_CLS_CONST = max(IMG_CLS_CONST, 1.0)
    # print(f' Ans: {batch["ans"].detach().cpu().tolist()}, ques: {batch["ques"].shape}, out: {out.shape}, pred: {pred.detach().cpu().tolist()}')

# losses = [3, 4, 2, 5]
    plt.clf()
    plt.plot(train_losses, color = 'r', label = 'train')
    plt.plot(val_losses, color = 'g', label = 'val')
    plt.title(f'Dataset: {datasetLength}, lambda: {lambda_}')
    plt.legend()
    plt.xlabel('Epochs')
    plt.ylabel('Loss')
    plt.savefig(os.path.join(EXP_DIR, 'LossGraph.png'))

    with open(os.path.join(EXP_DIR, 'losses.json'), 'w') as f:
        json.dump(
            {'word_accuracy': word_accuracy,
            'image_accuracy': image_accuracy,
            'losses': losses
            },
            f,
            indent=4
        )


import spacy
import copy
import debug

class Word:
    """
    Stores a single word with ID and Type
    """
    def __init__(self, word, wtype, wid):
        self.word = word
        self.word_type = wtype
        self.id = wid

class Understood:
    """
    Stores what was understood by the parser as in words found in the wordlist
    stores the ids based in the wordfile
    """
    def __init__(self):
        self.verb_id = 0
        self.dir_id = 0
        self.obj1_id = 0
        self.obj2_id = 0
        self.is_question = False

    def clear(self):
        self.verb_id = 0
        self.dir_id = 0
        self.obj1_id = 0
        self.obj2_id = 0
        self.is_question = False

class Adv_Parser:
    """
    Class to handle the parsing.
    """

    def __init__(self):
        """
        Initializes the blank word list and the spacy parser
        """
        self.wordlist = []
        self.nlp = spacy.load("de_core_news_sm")

    def load_words(self):
        """
        Loads word list which will be understood by the adventure
        """
        with open(r".\res\words.csv", "r", encoding="utf-8") as wordfile:
            # csv_file = csv.reader(wordfile, delimiter=";")
            for row in wordfile:
                a,b,c = row.split(";")
                w = Word(a, int(b), int(c))
                self.wordlist.append(w)

    def parse(self, command):
        verb = direction = obj1 = obj2 = adj = out = 0

        for w in command.split(" "):
            for lookup in self.wordlist:
                if w.lower() == lookup.word:
                    if lookup.word_type == 1:
                        verb = lookup.id
                    elif lookup.word_type == 2:
                        direction = lookup.id
                    elif lookup.word_type == 3:
                        if obj1 == 0:
                            obj1 = lookup.id
                        else:
                            obj2 = lookup.id
                    elif lookup.word_type == 4:
                        adj = lookup.id
                    else:
                        out = lookup.id

        return [verb, direction, obj1, obj2, adj, out]

    def lookup_word(self, w):
        """
        Looks up w in the wordlist and returns the type and the id if found otherwise 0,0
        :param w: word to lookup
        :return: type and id if found
        """
        for lookup in self.wordlist:
            if w.lower() == lookup.word:
                return [int(lookup.word_type), int(lookup.id)]

        return [0, 0]

    def parse_spacy(self, command):
        """
        Parses command using spacy returns list of commands
        ;param command: user entry for command
        ;return list of Understood
        """
        doc = self.nlp(command)

        return_commands = []

        ud = Understood()

        for token in doc:
            if token.pos_ in ["NOUN",  "VERB", "PUNCT", "CCONJ", "ADJ", "X", "ADV", "PART", "PROPN"]:      #ignore everything which is not a noun or a verb or a conjucntion
                if token.pos_ == "PUNCT" or token.lemma_ == "und":   # command end and add in case of interpunction or the word "und"
                                                                     # TODO put "und" in variable
                    ud.is_question = (token.lemma_ == "?")
                    if ud.verb_id != 0 or ud.dir_id != 0: # return if at least a verb or a direction was found
                        return_commands.append(copy.deepcopy(ud))
                    verb_save = ud.verb_id
                    ud.clear() # reset
                    ud.verb_id = verb_save          # in case of multiple objects save verb
                else:
                    wordtype, id = self.lookup_word(token.lemma_)

                    if debug.DEBUG:
                        print(f"typ {wordtype}, ID {id} {token.lemma_}")
                    if wordtype == 0:
                        wordtype, id = self.lookup_word(token.text)

                    if wordtype == 1:
                        ud.verb_id = id
                    if wordtype == 2:
                        ud.dir_id = id
                    if wordtype == 3: # store a maximum of two objects per command
                        if ud.obj1_id == 0:
                            ud.obj1_id = id
                        elif ud.obj2_id == 0:
                            ud.obj2_id = id
            if debug.DEBUG:
                print(token.text, token.lemma_, token.pos_, token.tag_, token.dep_, token.shape_, token.is_alpha, token.is_stop)
        if ud.verb_id != 0 or ud.dir_id != 0:  # return if at least a verb or a direction was found
            return_commands.append(copy.deepcopy(ud))
        return return_commands


import json
import urllib
import pandas as pd
import dash_html_components as html
import dash_bootstrap_components as dbc
import dash_core_components as dcc

from dash.dependencies import Input, Output, State
from app import app

def getInfo():
    urlData = 'https://data.bmkg.go.id/DataMKG/TEWS/gempaterkini.json'
    web = urllib.request.urlopen(urlData)
    if web.getcode() == 200:
        data = web.read()
        JSON = json.loads(data)

        dicts = []
        df_gempa = pd.DataFrame()

        for i in JSON['Infogempa']['gempa']:
            dicts.append(i)

        df_gempa = df_gempa.append(dicts, ignore_index=True)

        waktu_gempa = df_gempa['Tanggal'] + '\n' + df_gempa['Jam']

        df_gempa.insert(loc=0, column='Waktu Gempa', value=waktu_gempa)
        df_gempa.rename(columns={'Coordinates': 'Titik Gempa', 'Magnitude': 'Magnitudo'}, inplace=True)

        df_gempa = df_gempa.drop(['Tanggal', 'Jam', 'Lintang', 'Bujur', 'DateTime'], axis=1)
    else:
        print('Something wrong, cannot proceed')

    return df_gempa

def gempaTerbaru():
    urlData = 'https://data.bmkg.go.id/DataMKG/TEWS/autogempa.json'
    web = urllib.request.urlopen(urlData)
    if web.getcode() == 200:
        data = web.read()
        JSON = json.loads(data)

        wilayah = JSON['Infogempa']['gempa']['Wilayah']
        shakemap = 'https://ews.bmkg.go.id/TEWS/data/' + JSON['Infogempa']['gempa']['Shakemap']
        dirasakan = JSON['Infogempa']['gempa']['Dirasakan']
        waktu = JSON['Infogempa']['gempa']['Tanggal'] + ' - ' + JSON['Infogempa']['gempa']['Jam']
        magnitudo = JSON['Infogempa']['gempa']['Magnitude']
        kedalaman = JSON['Infogempa']['gempa']['Kedalaman']
        lokasi = JSON['Infogempa']['gempa']['Lintang'] + ' ' + JSON['Infogempa']['gempa']['Bujur']
        map = 'https://www.google.com/maps/search/' + JSON['Infogempa']['gempa']['Coordinates']

        terbaru = dbc.Card(
            dbc.CardBody(
                [
                    html.H5('Gempabumi Terkini', className='card-title'),
                    html.P(wilayah),
                    html.Label('Dirasakan (Skala MMI)'),
                    dbc.Badge(dirasakan, color='warning', className='mr-1'),
                    html.Br(),
                    dbc.Button('Detail Gempa', id='open', color='primary'),
                    dbc.Modal(
                        [
                            dbc.ModalHeader('Informasi Gempabumi'),
                            dbc.ModalBody([
                                    html.P('Peta Guncangan Gempabumi (Earthquake Shakemap)'),
                                    dbc.CardImg(src=shakemap, style={'width': '300px'}),
                                    dcc.Markdown(f'''
                                                **Parameter Gempabumi**
                                                {waktu}
                                                Magnitudo: {magnitudo}
                                                Kedalaman: {kedalaman}
                                                Lokasi: {lokasi}
                                                Google maps: [Lokasi Gempa]({map} 'Lokasi kejadian gempabumi')
                                                ''')
                            ]),
                            dbc.ModalFooter(
                                dbc.Button(
                                    'Tutup', id='close', className='ml-auto', n_clicks=0
                                )
                            ),
                        ],
                        id="modal",
                        is_open=False,
                    ),
                ]
            )
        )

    return terbaru

# layout web app
layout = html.Div([
    html.H1('Informasi Gempabumi'),
    html.Hr(),
    dbc.Row([
        dbc.Col(gempaTerbaru(), width='auto')
    ]),
    html.Br(),
    html.H3('Gempabumi Terkini (M≥5.0)'),
    dbc.Table.from_dataframe(getInfo(), striped=True, hover=True),
], style={'margin-top': '1%', 'margin-bottom': '1%', 'margin-left': '10%', 'margin-right': '10%'})

@app.callback(
    Output('modal', 'is_open'),
    [Input('open', 'n_clicks'), Input('close', 'n_clicks')],
    [State('modal', 'is_open')],
)
def toggle_modal(n1, n2, is_open):
    if n1 or n2:
        return not is_open
    return is_open


"""Define taskmanager commands tests."""

from io import StringIO

from django.core.management import call_command
from django.test import TestCase

from taskmanager.models import AppCommand

class CollectTaskCommandTest(TestCase):
    """A set of tests for collect task command."""

    def test_command_output(self):
        """Test the command output."""
        out = StringIO()
        call_command("collectcommands", stdout=out)
        self.assertIn("migrate", out.getvalue())

    def test_command_output_no_djangocore(self):
        """Test the command output invoked with excludecodre argument."""
        out = StringIO()
        call_command("collectcommands", "--excludecore", stdout=out)
        self.assertNotIn("migrate", out.getvalue())

    def test_command_creation(self):
        """Test the commands creation."""
        self.assertEqual(AppCommand.objects.all().count(), 0)
        out = StringIO()
        call_command("collectcommands", stdout=out)
        self.assertNotEqual(AppCommand.objects.all().count(), 0)


from geolocation import LocationError
from models import Location
import pytest

def test_location_latlong_gettering():
    location = Location(city_name="campinas", uf="sp")
    location.load_lat_lon()
    assert pytest.approx(location.lat, 0.01) == -22.89
    assert pytest.approx(location.lon, 0.01) == -47.10

def test_location_invalid_raises_error():
    location = Location(city_name="a test not a city", uf="sp")
    with pytest.raises(LocationError) as execinfo:
        location.load_lat_lon()
    assert "Geolocation not found" in str(execinfo.value)


from typing import List
from uuid import UUID

from fastapi import APIRouter, Depends, Response, status
from fastapi.exceptions import HTTPException
from structlog import get_logger

from fastapi_server.lib.logs import exception_to_string
from fastapi_server.resources.database.postgres import get_db_session
from fastapi_server.resources.database.database import get_databases

from fastapi_server.apps.users.schemas import ActiveUser
from fastapi_server.apps.users.auth import with_active_user
from ..schemas import (
    ProjectMemberCreate,
    ProjectMemberRead,
    ProjectMemberUpdate,
    ProjectRead,
    ProjectCreate,
    ProjectUpdate,
)
from ..database.postgres import ProjectsDB, get_projects_db
from ..entitlements import project_with_permissions
from .. import core

from .schemas import CreateProjectMemberPayload, CreateProjectResponse

logger = get_logger("projects")

router = APIRouter(
    prefix="/projects", tags=["projects"], responses={404: {"description": "Not found"}}
)

@router.get(
    "/projects",
    name="projects:projects:get-projects",
    response_description="Get all Projects",
    response_model=List[ProjectRead],
)
async def get_projects(
    db: ProjectsDB = Depends(get_projects_db),
    user: ActiveUser = Depends(with_active_user),
):
    projects = await db.get_projects()
    logger.info(
        "Get Projects",
        code="projects-get_projects",
        user_id=user.user_id,
        projects=len(projects),
    )
    return projects

@router.get(
    "/projects/user",
    name="projects:projects:get-user-projects",
    response_description="Get user Projects",
    response_model=List[ProjectRead],
)
async def get_user_projects(
    db: ProjectsDB = Depends(get_projects_db),
    user: ActiveUser = Depends(with_active_user),
):
    projects = await db.get_projects_by_user(user.user_id)
    logger.info(
        "Get User Projects",
        code="projects-get_user_projects",
        user_id=user.user_id,
        projects=len(projects),
    )
    return projects

@router.post(
    "/projects",
    name="projects:projects:add-project",
    status_code=status.HTTP_201_CREATED,
)
async def create_project(
    project_create: ProjectCreate,
    db: ProjectsDB = Depends(get_projects_db),
    user: ActiveUser = Depends(with_active_user),
) -> CreateProjectResponse:
    created_project, created_project_member = await core.create_project(
        db, user.user_id, project_create
    )
    logger.info(
        "Create project",
        code="projects-create_project",
        user_id=user.user_id,
        project_id=str(created_project.id),
    )
    return {"project": created_project, "project_member": created_project_member}

@router.patch(
    "/projects/{project_id}",
    name="projects:projects:update-project",
    response_model=ProjectRead,
)
async def update_project(
    project_id: UUID,
    project_update: ProjectUpdate,
    db: ProjectsDB = Depends(get_projects_db),
    user: ActiveUser = Depends(with_active_user),
):
    log_kwargs = {
        "code": "projects-update_project",
        "user_id": user.user_id,
        "project_id": str(project_id),
    }
    await project_with_permissions(db, project_id, user.user_id, True)
    try:
        updated_project = await core.update_project(
            db, user.user_id, project_id, project_update
        )
    except Exception as e:
        logger.error("Update project", exception=exception_to_string(e), **log_kwargs)
        raise HTTPException(
            status_code=status.HTTP_400_BAD_REQUEST,
        )
    logger.info("Update project", **log_kwargs)
    return updated_project

@router.delete(
    "/projects/{project_id}",
    name="projects:projects:delete-project",
    status_code=status.HTTP_204_NO_CONTENT,
    response_class=Response,
)
async def delete_project(
    project_id: UUID,
    db: ProjectsDB = Depends(get_projects_db),
    user: ActiveUser = Depends(with_active_user),
):
    log_kwargs = {
        "code": "projects-delete_project",
        "user_id": user.user_id,
        "project_id": str(project_id),
    }
    await project_with_permissions(db, project_id, user.user_id, True)
    try:
        await core.delete_project(db, project_id)
    except Exception as e:
        logger.error("Delete project", exception=str(e), **log_kwargs)
        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST)
    logger.info("Delete project", **log_kwargs)

    return None

"""
PROJECT MEMBERS
"""

@router.get(
    "/projects/{project_id}/members",
    name="projects:project-members:get-project-members",
    response_description="Get Project Members",
    response_model=List[ProjectMemberRead],
)
async def get_project_members(
    project_id: UUID,
    db: ProjectsDB = Depends(get_projects_db),
    user: ActiveUser = Depends(with_active_user),
):
    await project_with_permissions(db, project_id, user.user_id)
    project_members = await db.get_project_members_by_project_id(project_id)
    if not project_members:
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
    return project_members

@router.post(
    "/projects/{project_id}/members",
    name="projects:project-members:create-project-member",
    response_description="Create Project Member",
    response_model=ProjectMemberRead,
    status_code=status.HTTP_201_CREATED,
)
async def create_project_member(
    project_id: UUID,
    project_member_create_payload: CreateProjectMemberPayload,
    session=Depends(get_db_session),
    user: ActiveUser = Depends(with_active_user),
):
    dbs = get_databases(session)
    await project_with_permissions(dbs["projects"], project_id, user.user_id, True)
    try:
        user_to_add = await dbs["users"].get_user_by_email(
            project_member_create_payload.user_email
        )
        if not user_to_add:
            raise Exception()

        project_member_create = ProjectMemberCreate(
            project_id=project_id,
            user_id=user_to_add.id,
            is_admin=project_member_create_payload.is_admin,
        )
        project_member = await core.create_project_member(
            dbs["projects"], user.user_id, project_member_create
        )
    except Exception as e:
        logger.error(
            f"Create project member for project {project_id}",
            project_id=str(project_id),
            user_id=user.user_id,
            project_member_create_payload=project_member_create_payload.model_dump(),
            exception=exception_to_string(e),
        )
        raise HTTPException(
            status_code=status.HTTP_400_BAD_REQUEST,
        )
    return project_member

@router.patch(
    "/projects/{project_id}/members/{member_id}",
    name="projects:project-members:update-project-member",
    response_model=ProjectMemberRead,
)
async def update_project_member(
    project_id: UUID,
    member_id: str,
    project_member_update: ProjectMemberUpdate,
    db: ProjectsDB = Depends(get_projects_db),
    user: ActiveUser = Depends(with_active_user),
):
    await project_with_permissions(db, project_id, user.user_id, True)

    try:
        updated_project_member = await core.update_project_member(
            db, user.user_id, project_id, member_id, project_member_update
        )
    except Exception as e:
        logger.error(
            f"Update project member {project_id}",
            project_id=project_id,
            user_id=user.user_id,
            member_id=member_id,
            project_member=project_member_update.model_dump(),
            exception=exception_to_string(e),
        )
        raise HTTPException(
            status_code=status.HTTP_400_BAD_REQUEST,
        )

    return updated_project_member

@router.delete(
    "/projects/{project_id}/members/{member_id}",
    name="projects:project-members:remove-project-member",
    response_description="Remove Project Member",
    status_code=status.HTTP_204_NO_CONTENT,
)
async def remove_project_member(
    project_id: UUID,
    member_id: str,
    db: ProjectsDB = Depends(get_projects_db),
    user: ActiveUser = Depends(with_active_user),
):
    await project_with_permissions(db, project_id, user.user_id, True)
    try:
        await db.delete_project_member_by_user_id(member_id)
    except Exception as e:
        logger.error(
            f"Delete project member for {project_id}",
            project_id=str(project_id),
            member_id=member_id,
            user_id=user.user_id,
            exception=exception_to_string(e),
        )
        raise HTTPException(
            status_code=status.HTTP_400_BAD_REQUEST,
        )
    return None


import urllib.request
import json
import pprint  # module that makes it easier to print and look at dictionaries

# Challenge: write a program that prints all of the status updates

search_term = input('Please enter a search term (defaults to "northwestern"): ')
if search_term == '':
    search_term = 'Northwestern'
url = 'https://eecs110-twitter-proxy.herokuapp.com/1.1/search/tweets.json?q='
url += search_term
request = urllib.request.urlopen(url)
data = json.loads(request.read().decode())

pprint.pprint(data, depth=1)


# coding: utf-8
#Imports {{{1
from os import path
from datetime import date
import logging
# app engine
import webapp2
from google.appengine.api import users
from google.appengine.ext import db
# Шаблонизатор mako
from mako.template import Template
from mako.lookup import TemplateLookup
TEMPLATES_DIR = path.join(path.dirname(__file__), 'templates')
lookup = TemplateLookup(
    directories=[TEMPLATES_DIR],
    input_encoding="utf-8",
    format_exceptions=True,
)
# проект
from models import *
import forms
from profile import build_residence
from lib.functions import get_i18n

# Global variables {{{1
empty_values = (None, '', []) # значения полей, при которых оно рассматривается как пустое

#class FillData(webapp2.RequestHandler) {{{1
class FillData(webapp2.RequestHandler):
    """/prof/fill_data"""
    #def __init__(self, *args, **kwargs) {{{2
    def __init__(self, *args, **kwargs):
        super(FillData, self).__init__(*args, **kwargs)
        self.WARNINGS = []

    def get(self, action=None, form=None, locationForm=None, Date=None): #{{{2
        """Предоставить пользователю форму для заполнения параметров.
        Отправляются только те данные, которые заполнены.
        form - форма с заполненными данными и ошибками
        Date - дата, за которую показывать значения
        """
        # locale
        i18n = get_i18n(self.request)
        _ = i18n.gettext
        lang = i18n.locale

        # animation
        animation = self.request.cookies.get('animation', True)
        if animation == 'false':
            animation = False
        else: # 'true' или что-то странное
            animation = True

        action_url = '/prof/fill_data'
        user = users.get_current_user()

        if user:    # Пользователь залогинен
            # Проверить, есть ли данный пользователь в нашем хранилище
            U = User.get_by_key_name(user.email())
            # ссылка для разлогинивания
            logout_url = users.create_logout_url(self.request.uri)

            # Если пользователь зарегистрирован
            if U:
                if not form:
                    form = forms.FillDataForm(U, Date=Date)
                residence = U.residence.name
                if locationForm is None:
                    locationForm = forms.LocationForm()

                template_values = {
                    '_': _,
                    'lang': lang,
                    'animation': animation,
                    'user': user, # login.mako
                    'logout_url': logout_url, # login.mako
                    'route': '/prof/fill_data',
                    'WARNINGS': self.WARNINGS,
                    'form': form,
                    'locationForm': locationForm,
                    'residence': residence,
                    'action_url': action_url,
                    'action': action
                }

                # Формируем и рендерим шаблон
                template = lookup.get_template("fill_data.mako")
                self.response.write(template.render(**template_values))
            # Если пользователь не зарегистрирован
            else:
                self.redirect("/prof")
        # Пользователь не залогинен
        else:
            self.redirect(users.create_login_url(self.request.uri))

    #def post(self) {{{2
    def post(self):
        """Заполнение тренда"""
        user = users.get_current_user()
        today = date.today()
        # два фиксированных параметра
        if user:
            U = User.get_by_key_name(user.email())
        else:
            return self.get()

        # Нажата кнопка "Сохранить" при выборе населённого пункта
        if self.request.get('submitLocationForm'):
            # получаем объект residence
            lang = self.request.get('lang')
            country = self.request.get('country')
            region = self.request.get('region')
            district = self.request.get('district')
            try:
                residence = build_residence(lang, country, region, district)
            except (IOError, KeyError): # Несовместимые пункты в select'ах
                return self.get()
            # строим форму
            form = forms.LocationForm(residence, formdata=self.request.POST)
            if form.validate():
                if form.locality.data: # если был запрос на изменение места жительства
                    name, id, url = form.locality.data.rsplit(';', 2)
                    keyname = "id_%s" % id
                    R = Residence.get_or_insert(keyname, id=int(id), name=name, url=url)
                    # Если м.ж. поменялось и у прежнего м.ж. это единственный пользователь, удаляем
                    if R.key() != U.residence.key() and U.residence.users.count() < 2:
                        U.residence.delete()
                    # Обязательные поля
                    U.residence = R
                    U.put()

                return self.get()
            else:
                return self.get(locationForm=form, action='editLocation')

        # Нажата кнопка "Обновить значения параметров"
        elif self.request.get('update_data'):
            form = forms.FillDataForm(U, formdata=self.request.POST)
            logging.warning(self.request.get('date'))

            if form.validate(): # Проверка пройдена (правда нужна только дата)
                Date = form.date.data
                return self.get(form=None, Date=Date)
            else:
                return self.get(form=form)

        # Нажата кнопка 'Занести значения'
        elif self.request.get('submit'):
            form = forms.FillDataForm(U, formdata=self.request.POST)
            logging.warning(self.request.get('date'))

            if form.validate(): # Проверка пройдена
                Date = form.date.data
                keyname = "%s_%s" % (U.nickname, Date.strftime("%Y%m%d"))
                UT = UserTrend.get_by_key_name(keyname, parent=U)
                if not UT:
                    ccd_key = "%s-%s" % (U.residence.key().name(), Date.strftime("%Y%m%d"))
                    UT = UserTrend(key_name=keyname, parent=U, date=today, ccd_key=ccd_key)

                all_values_are_empty = True # вначале все значения пустые

                # Подмена id не пройдёт, т.к. значения параметров и элементов берутся из валидированной формы
                for field in form:
                    if field.data not in empty_values:  # если значение есть
                        # 2 вида полей первого порядка (обычные и FormField)
                        if not hasattr(field, 'form'):  # обычный параметр
                            if field.name not in ('UTCdate', 'isToday'):
                                setattr(UT, field.name, field.data)
                                if field.name != 'date':
                                    all_values_are_empty = False
                        else:   # FormField (сложные параметры)
                            for subfield in field:
                                subfield_name = subfield.name.split('_')[1]
                                if subfield.data not in empty_values:   # если значение есть
                                    setattr(UT, subfield_name, subfield.data)
                                    all_values_are_empty = False
                                else:
                                    # если значение было, то свойство удаляется
                                    value = getattr(UT, subfield_name, None)
                                    if value not in empty_values:
                                        delattr(UT, subfield_name)
                    else:
                        # если значение было, то свойство удаляется
                        value = getattr(UT, field.name, None)
                        if value not in empty_values:
                            delattr(UT, field.name)

                if all_values_are_empty == True:
                    # пользователь не заполнил либо стёр данные, удаляем запись в хранилище
                    self.WARNINGS.append(u"Вы не заполнили данные")
                    db.delete(UT)
                else:
                    UT.put()
                    self.WARNINGS.append(u"Значения добавлены")

                return self.get(Date=Date)
            else:
                return self.get(form=form)


import configparser
import logging
import re
from cgitb import text
from tkinter import NO, E

class Translator:
    def __init__(self) -> None:
        self.map = {}

    def _read_config(self) -> configparser.ConfigParser:
        config = configparser.ConfigParser()
        with open("./config/settings.cfg") as f:
            config_text = f.read()
            config.read_string(config_text)
        return config

    def _is_contain_chinese(self, word):
        pattern = re.compile(r"[\u4e00-\u9fa5]")
        match = pattern.search(word)
        return True if match else False

    def _get_cache(self, text):
        if self.map.__contains__(text):
            return self.map[text]
        else:
            return None

    def _cache(self, cn_text, en_text):
        if not self.map.__contains__(text):
            self.map[cn_text] = en_text

    def _cn2en(self, text):
        raise NotImplemented

    def cn2en(self, text):
        if not self._is_contain_chinese(text):
            return text
        if self._get_cache(text) is not None:
            return self._get_cache(text)
        try:
            en_text = self._cn2en(text)
            self._cache(text, en_text)
            text = en_text
        except Exception as e:
            logging.error(f"{text} translate error: {e}")
        return text


from matplotlib import pyplot as plt
from matplotlib.colors import ListedColormap

from mne.filter import filter_data

from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC, LinearSVC
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import KFold
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from sklearn.decomposition import PCA
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import make_moons, make_circles, make_classification
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis

import scipy.signal as scisig
import numpy as np
import pandas as pd
import csv
import pickle
import datetime

def ConvertEasyToDataFrame(filename):
    """ This function converts a single .easy file (from NIC2) to an easy-to-use dataframe.
    Uses both the .easy file and .info file (containing metadata)

    ---- Input ----
    filename: string containing the .easy filepath

    ---- Output ----
    df: dataframe containing all the EEG, accelerometer, and event marker data
    fs: sampling rate for the EEG data (Hz)
    fs_accel: sampling rate for the accelerometer data (Hz)

    """
    import pandas as pd

    # Read in the .easy file
    df = pd.read_csv(filename, delimiter='\t', header=None)

    # Get metadata from the .info file
    fname = filename[:-5] + '.info'
    with open(fname) as f:
        content = f.readlines()
    content = [x.strip() for x in content]

    # Get the channel names
    channel_info = [x for x in content if 'Channel ' in x]
    channel_names = []
    for ch in range(len(channel_info)):
        channel_names.append(channel_info[ch].split(': ')[1])

    channel_names.append('X')
    channel_names.append('Y')
    channel_names.append('Z')
    channel_names.append('STI 014')
    channel_names.append('DateTime')

    # Get sampling rates
    sampling_rates = [x for x in content if 'sampling rate: ' in x]
    fs_all = []
    for freq in range(len(sampling_rates)):
        fs_all.append(float(sampling_rates[freq].split(': ')[1].split(' ')[0]))

    # Store sampling rates
    fs = fs_all[0]
    fs_accel = fs_all[1]

    # Assign the column names
    df.columns = channel_names

    # Return dataframe and sampling rates
    return df, fs, fs_accel

def PlotAverageSpectrogram(epochs, trials, chanOfInt, fs, ds_factor):
    """
    Input
    epochs: your data epochs (raw)
    trials: list or array of trials you wish to include in the average (e.g., all trials of type A)
    chanOfInt: name of the channel of interest
    fs: sampling rate
    ds_factor: downsampling factor
    """

    Sxx_all = list()

    for trialOfInt in trials:
        f, t, Sxx = scisig.spectrogram(scisig.decimate(epochs[trialOfInt][chanOfInt], ds_factor), fs/ds_factor, nperseg=10)
        # nperseg determines size of time bins, number of time bins = fs/nperseg - 1
        # nperseg = number of segments per second, so 10 = 100 ms chunks
        Sxx_all.append(Sxx)

    Sxx_avg = np.mean(Sxx_all,0)

    return f, t, Sxx_avg

if __name__ == "__main__":
    subjID = input('Subject ID: ')
    EEGdevice = int(input('7 if DSI-7, 8 if Enobio: '))
    filepath_EEG = input('EEG data file path (.easy or .csv): ')
    filepath_behavioral = input('Behavioral data filepath (.csv): ')

    ## Load in data from the EEG file
    if EEGdevice == 7:
        EEGmeta = list()
        # Open and read ('r') csv file with raw data
        with open(filepath_EEG, 'r') as csvfile:
            for row in range(0,15):
                EEGmeta.append(csvfile.readline())

        # Save sampling frequency
        fs = int(EEGmeta[1].split(',')[1])

        # Skip the first 15 rows that contain the meta data
        EEGdata = pd.read_csv(filepath_EEG,',',skiprows=15)

    elif EEGdevice == 8:
        EEGdata, fs, fs_accel = ConvertEasyToDataFrame(filepath_EEG)

    else:
       raise ValueError('Invalid EEG device number')

    # Load behavioral data
    behavioralData = pd.read_csv(filepath_behavioral, ',')
    behavioralData = behavioralData.transpose()

    # Saving header information
    behavioralHeader = behavioralData.iloc[0]

    behavioralData = behavioralData.iloc[2:]
    behavioralData.columns = behavioralHeader

    ## Get event markers
    print('Get event markers')
    if EEGdevice == 7:
        # Uses audio pulses
        # Store where the values in trigger are equal to 8 (the audio trigger input channel number)
        index_trigger = np.where(EEGdata['Trigger']==8)

        # Initial peak/jump detection (note that it isn't clean all the time so just take initial)
        index_trigger = np.where(np.diff(EEGdata['Trigger'])>1) #use EEGdata['Trigger'] to make sure we get initial

        # Delete suspicious trigger pulses that are closer than 4 seconds to one another
        index_trigger = np.delete(index_trigger, np.where(np.diff(index_trigger[0]) < (4*fs)))

        # Remove initial sync pulse (there were two in this case)
        index_trigger = index_trigger[2:]

        # Check number of trials
        num_of_trials = behavioralData.shape[0]
        channels = EEGdata.columns[1:8]

        # If the number of trials is greater than number of pulses,
        # Likely last trial pulse was not sent, so estimate it
        if np.shape(index_trigger)[0] < num_of_trials:
            index_trigger = np.append(index_trigger, int(index_trigger[-1]+np.mean(np.diff(index_trigger))))

    elif EEGdevice == 8:
        # Use LSL event markers
        index_trigger = np.where(EEGdata['STI 014']!=0)
        index_trigger = index_trigger[0]
        num_of_trials = behavioralData.shape[0]
        channels = EEGdata.columns[0:7]

        if num_of_trials > len(index_trigger):
            # Task was run with first trial sending pulse of value 0
            # so remove the first trial since we don't know when it starts
            num_of_trials = num_of_trials - 1

    ## Separate into epochs
    print('Separate into epochs')
    trialLength = int(behavioralData['trialLength'][0])

    # Get trial timing
    t_trial_start = list()
    t_trial_end = list()

    # Creating lists of all trigger start and end locations
    for i in range(0,num_of_trials):
        t_trial_start.append(index_trigger[i])
        t_trial_end.append(index_trigger[i] + int(trialLength*fs))

    # Save rest period epochs as well as trials for comparison
    t_rest_start = list()
    t_rest_end = list()

    for i in range(num_of_trials-1):
        t_rest_start.append(t_trial_end[i])
        t_rest_end.append(t_trial_start[i+1])

    ## Cleaning EEG data
    print('Clean EEG data')
    EEGdata_filt = EEGdata.copy()
    eeg_data = EEGdata[channels].values * 1.0 # multiply by 1.0 to convert int to float
    filtered = filter_data(eeg_data.T, sfreq=fs, l_freq=1, h_freq=40)
    EEGdata_filt[channels] = filtered.T

    epochs_raw = []
    epochs_filt = []

    for trial in range(0,len(t_trial_start)):
        epochs_raw.append(EEGdata.loc[t_trial_start[trial]:t_trial_end[trial]][channels])
        epochs_filt.append(EEGdata_filt.loc[t_trial_start[trial]:t_trial_end[trial]][channels])

    ## Organize trials
    print('Organize trials')
    # Create lists for each trial type
    trialL = list()
    trialR = list()
    i = 0

    for letter in behavioralData['trialType'][1:]:
        if letter == 'L':
            trialL.append(i)
        elif letter == 'R':
            trialR.append(i)
        elif letter == 'B':
            trialR.append(i)
        elif letter == 'D':
            trialR.append(i)
        i += 1

    # Create a single list that includes which trial is which
    trial_type = list()
    i = 0

    for letter in behavioralData['trialType'][1:]:
        if letter == 'L':
            trial_type.append(0)
        elif letter == 'R':
            trial_type.append(1)
        elif letter == 'B':
            trial_type.append(0)
        elif letter == 'D':
            trial_type.append(1)
        i += 1

    ## Store the data
    print('Store the data')
    # Store spectrogram data for each trial, for each channel
    ds_factor = 4 # downsampling factor
    channels_to_use = ['C3', 'C4'] # to use all channels, use channels
    epochs_Sxx = [[]]*len(channels_to_use)
    i = 0

    for chanOfInt in channels_to_use:
        tmp = list()
        for trial in range(0,len(trial_type)):
            f, t, Sxx = scisig.spectrogram(scisig.decimate(epochs_raw[trial][chanOfInt].T, ds_factor), fs/ds_factor, nperseg=10)
            tmp.append(Sxx.flatten('C')) # if you want all frequency data
            #tmp.append(Sxx[1]) # if you just want second frequency bin (10-20 Hz if you do ds_factor = 3)
        epochs_Sxx[i] = tmp
        i += 1

    # Format for classification
    print('Format data for classification')
    epochs_collapsed = []

    # Concatenate all epochs such that shape is (num of trials x (flattened channels and spec data))
    # So this should be 100 x 2310 (which is 7 channels * 330 spectrogram data points)
    num_of_channels = np.shape(epochs_Sxx)[0]
    num_of_trials = np.shape(epochs_Sxx)[1]
    num_of_specData = np.shape(epochs_Sxx)[2]

    epochs_collapsed = np.asarray(epochs_Sxx).reshape(num_of_trials, num_of_channels * num_of_specData)

    h = .02  # step size in the mesh

    # preprocess dataset, split into training and test part
    print('Preprocess data for classification')
    X = [np.mean(epochs_collapsed,1),np.std(epochs_collapsed,1)]
    X = np.asarray(X).T
    y = trial_type
    args = np.arange(len(X))
    np.random.shuffle(args)
    X = [X[i] for i in args]
    y = [y[i] for i in args]
    X = StandardScaler().fit_transform(X)
    X_train, X_test, y_train, y_test = \
        train_test_split(X, y, test_size=.4, random_state=42)

    x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
    y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                        np.arange(y_min, y_max, h))

    ## Classify
    print('Classify')
    name = "Random Forest"
    clf = RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1)
    clf.fit(X_train, y_train)
    score = clf.score(X_test, y_test)
    print('Score: ' + str(score))

    # Save model
    print('Save model')
    model = clf
    time_to_save = datetime.datetime.now().isoformat()
    time_to_save = time_to_save.replace('T','-')
    time_to_save = time_to_save.replace(':','-')
    model_file = subjID + '_MI_classifier_' + time_to_save[:19] + '.sav'
    pickle.dump(model, open(model_file, 'wb'))

    # Save training/testing examples
    print('Save training/testing examples')
    filepath_export_data = 'data_for_training_' + time_to_save[:19] + '.npz'
    np.savez_compressed(filepath_export_data, X=X, y=y)


import streamlit as st
state = st.session_state
import pandas as pd
import base64
import requests
from scripts import tools

tools.page_config()
tools.get_data()

head_cols = st.columns((7,2,1))
with head_cols[0]:
    st.write('# Good for your heart foods')
    st.write('A guide to heart healthy vegetarian foods that can help to lower blood pressure.')
with head_cols[1]:
    st.image(tools.round_corners('res/head_img.jpg'), width=400)

st.write('## Raw foods')
rf_cols = st.columns((5,1))
with rf_cols[0]:
    st.write('*See the nutritional value of some common foods. The progress bar displays the value in relation to the other foods in the list. Click any column header to change the sort order.*')
with rf_cols[1]:
    hide_pb_rf = st.checkbox('Hide progress bars', key="hide_pb_rf")

if hide_pb_rf:
    st.dataframe(state.ing_df, use_container_width=True)
else:
    st.data_editor(state.ing_df,column_config={
                c: st.column_config.ProgressColumn(
                format="%f", min_value=float(state.ing_df[c].min()),
                max_value=float(state.ing_df[c].max())) for c in state.ing_df},
                use_container_width=True)

st.write('## Prepared foods')
meal_tabs1, meal_tabs2 = st.tabs(['Nutritional value', 'Recipe ideas'])

with meal_tabs1:

    pf_cols = st.columns((5,1))
    with pf_cols[0]:
        st.write('*See the nutritional value of some prepared foods. The progress bar displays the value in relation to the other foods in the list. Click any column header to change the sort order.*')
    with pf_cols[1]:
        hide_pb_pf = st.checkbox('Hide progress bars', key="hide_pb_pf")

    if hide_pb_pf:
        st.dataframe(state.prep_df, use_container_width=True)
    else:
        st.data_editor(state.prep_df,column_config={
                    c: st.column_config.ProgressColumn(
                    format="%d", min_value=float(state.prep_df[c].min()),
                    max_value=float(state.prep_df[c].max())) for c in state.prep_df},
                    use_container_width=True)

with meal_tabs2:

    st.write('## Meal ideas')
    ing_search_cols = st.columns((3,1,4))
    with ing_search_cols[0]:
        ing_filter = st.text_input('*Filter by ingredient (separate multiple ingredients with a comma)*',
                                    key="ing_filter")
    if ing_filter.strip() != '':
        ings = ing_filter.split(',')

        try:
            ings = [tools.get_lemma(i) for i in ings]
        except:
            pass

        conditions = [state.meals_df['ingredients'].str.contains(i.strip(), case=False) for i in ings]
        state.meals_df_filtered = state.meals_df[pd.DataFrame(conditions).all(axis=0)]

        # state.meals_df = state.meals_df[
        #                 state.meals_df.ingredients.str.contains('|'.join([i.strip() \
        #                 for i in ing_filter.split(',')]), case=False)]
    else:
        state.meals_df_filtered = state.meals_df
    with ing_search_cols[1]:
        st.button("clear text input", on_click=tools.clear_text)

    meal_cols = st.columns(4)
    ct = 0
    for i,r in state.meals_df_filtered.iterrows():

        with meal_cols[ct]:
            try:
                img = requests.get(r.image)
                img.raise_for_status()  # Check if the request was successful
                thumb = img.content
            except requests.RequestException as e:
                print(f"Error fetching the image: {e}")
            st.markdown(
                    f"""

{r.meal}
""",unsafe_allow_html=True,)

            # st.write(html, unsafe_allow_html=True)
            if ct < 3:
                 ct += 1
            else:
                ct = 0

with st.expander('Sources'):
    st.write('Dataset: Food suitable for diabetes and blood pressure. [https://www.kaggle.com/datasets/nandagopll/food-suitable-for-diabetes-and-blood-pressure](https://www.kaggle.com/datasets/nandagopll/food-suitable-for-diabetes-and-blood-pressure)')