#! /usr/bin/env python

from __future__ import annotations
import numpy as np
import time
import math
from termcolor import colored
from typing import List, Dict, Tuple

class Utilities(object):

	@staticmethod
	def q_from_pol(mdp: HighDimMDP, pol: np.ndarray, task_idx: int = None) -> np.ndarray:
		X = mdp.states
		A = mdp.actions
		P = mdp.transitions_prob
		if task_idx is not None:
			c = mdp.costs[task_idx]
		else:
			c = mdp.costs
		gamma = mdp.gamma
		nX, nA = len(X), len(A)

		q = np.zeros((nX, nA))
		J = mdp.evaluate_pol(pol, task_idx)

		for act in range(nA):
			p = np.zeros((nX, nX))
			for state_idx in range(nX):
				state_tansitions = P[A[act]][state_idx]
				for state in state_tansitions:
					p[state_idx, int(state[0])] += state[1]
			q[:, act, None] = c[:, act, None] + gamma * p.dot(J)

		return q

	@staticmethod
	def v_from_q(q: np.ndarray, pol: np.ndarray) -> np.ndarray:
		return (q * pol).sum(axis=1)

	@staticmethod
	def Ppi(transitions: Dict[str, np.ndarray], actions: List[str], pol: np.ndarray) -> np.ndarray:
		nX = len(transitions[actions[0]])
		nA = len(actions)
		ppi = np.zeros((nX, nX))
		for act_idx in range(nA):
			for state_idx in range(nX):
				state_tansitions = transitions[actions[act_idx]][state_idx]
				for state in state_tansitions:
					ppi[state_idx, int(state[0])] += pol[state_idx, act_idx] * state[1]

		return ppi

	@staticmethod
	def softmax(param: np.ndarray, temp: float) -> np.ndarray:
		tmp = np.exp((param - np.max(param, axis=1)[:, None]) / temp)
		return tmp / tmp.sum(axis=1)[:, None]

	@staticmethod
	def softmax_grad(param: np.ndarray, temp: float) -> np.ndarray:
		softmax_pol = Utilities.softmax(param, temp)
		nX, nA = softmax_pol.shape
		sofmax_grad = np.zeros((nX * nA, nX * nA))

		for x in range(nX):
			for a in range(nA):
				for a2 in range(nA):
					sofmax_grad[x * a, x * a2] = softmax_pol[x, a] * ((1 if a == a2 else 0) - softmax_pol[x, a2])

		return np.diagonal(sofmax_grad).reshape((nX, nA), order='F')

	@staticmethod
	def get_goal_state(states: np.ndarray, goal: Tuple, obj_locs: List[Tuple]) -> List[int]:
		state_lst = list(states)
		goal_loc = ''
		for obj_loc in obj_locs:
			if obj_loc[2] in goal:
				goal_loc += str(obj_loc[0]) + ' ' + str(obj_loc[1]) + ', '
		return [state_lst.index(x) for x in states if x.find(goal_loc[:-2]) != -1]

class HighDimMDP(object):

	def __init__(self, x: np.ndarray, a: List[str], p: Dict[str, np.ndarray], c: np.ndarray, gamma: float, goal_states: List[int], feedback_type: str,
				 verbose: bool):
		self._mdp = (x, a, p, c, gamma)
		self._goal_states = goal_states
		self._verbose = verbose
		if feedback_type.lower().find('cost') != -1 or feedback_type.lower().find('reward') != -1:
			self._feedback_type = feedback_type
		else:
			print('Invalid feedback type, defaulting to use costs.')
			self._feedback_type = 'cost'

	@property
	def states(self) -> np.ndarray:
		return self._mdp[0]

	@property
	def actions(self) -> List[str]:
		return self._mdp[1]

	@property
	def transitions_prob(self) -> Dict[str, np.ndarray]:
		return self._mdp[2]

	@property
	def costs(self) -> np.ndarray:
		return self._mdp[3]

	@property
	def gamma(self) -> float:
		return self._mdp[4]

	@property
	def goals(self) -> List[int]:
		return self._goal_states

	@property
	def mdp(self) -> Tuple[np.ndarray, List[str], Dict[str, np.ndarray], np.ndarray, float]:
		return self._mdp

	@property
	def verbose(self) -> bool:
		return self._verbose

	@mdp.setter
	def mdp(self, mdp: Tuple[np.ndarray, List[str], Dict[str, np.ndarray], np.ndarray, float]):
		self._mdp = mdp

	@goals.setter
	def goals(self, goals: List[str]):
		self._goal_states = goals

	@verbose.setter
	def verbose(self, verbose: bool):
		self._verbose = verbose

	def get_possible_states(self, q: np.ndarray) -> np.ndarray:
		nonzerostates = np.nonzero(q.sum(axis=1))[0]
		possible_states = [np.delete(nonzerostates, np.argwhere(nonzerostates == g)) for g in self._goal_states][0]
		return possible_states

	def evaluate_pol(self, pol: np.ndarray, task_idx: int = None) -> np.ndarray:
		X = self._mdp[0]
		A = self._mdp[1]
		P = self._mdp[2]
		if task_idx is None:
			c = self._mdp[3]
		else:
			c = self._mdp[3][task_idx]
		gamma = self._mdp[4]

		nS = len(X)

		# Cost and Probs averaged by policy
		cpi = (pol * c).sum(axis=1)
		ppi = Utilities.Ppi(P, A, pol)

		# J = (I - gamma*P)^-1 * c
		J = np.linalg.inv(np.eye(nS) - gamma * ppi).dot(cpi)

		return J[:, None]

	def value_iteration(self, task_idx: int = None) -> np.ndarray:
		X = self._mdp[0]
		A = self._mdp[1]
		P = self._mdp[2]
		if task_idx is not None:
			c = self._mdp[3][task_idx]
		else:
			c = self._mdp[3]
		gamma = self._mdp[4]

		nX = len(X)
		nA = len(A)

		J = np.zeros(nX)
		err = 1
		i = 0

		while err > 1e-8:
			Q = []
			for act in range(nA):
				p = np.zeros((nX, nX))
				for state_idx in range(nX):
					state_tansitions = P[A[act]][state_idx]
					for state in state_tansitions:
						p[state_idx, int(state[0])] += state[1]
				Q += [c[:, act] + gamma * p.dot(J)]

			if self._feedback_type.find('cost') != -1:
				Jnew = np.min(Q, axis=0)
			else:
				Jnew = np.max(Q, axis=0)
			err = np.linalg.norm(J - Jnew)
			J = Jnew

			i += 1

		return J[:, None]

	def policy_iteration(self, task_idx: int = None) -> (np.ndarray, np.ndarray):
		X = self._mdp[0]
		A = self._mdp[1]
		P = self._mdp[2]
		if task_idx is not None:
			c = self._mdp[3][task_idx]
		else:
			c = self._mdp[3]
		gamma = self._mdp[4]

		nX = len(X)
		nA = len(A)

		# Initialize pol
		pol = np.ones((nX, nA)) / nA

		# Initialize Q
		Q = np.zeros((nX, nA))

		quit = False
		i = 0

		while not quit:
			if self._verbose:
				print('Iteration %d' % (i + 1), end='\r')

			J = self.evaluate_pol(pol, task_idx)

			for act in range(nA):
				p = np.zeros((nX, nX))
				for state_idx in range(nX):
					state_tansitions = P[A[act]][state_idx]
					for transition in state_tansitions:
						p[state_idx, int(transition[0])] += transition[1]
				Q[:, act, None] = c[:, act, None] + gamma * p.dot(J)

			if self._feedback_type.find('cost') != -1:
				Qbest = Q.min(axis=1, keepdims=True)
			else:
				Qbest = Q.max(axis=1, keepdims=True)
			polnew = np.isclose(Q, Qbest, atol=1e-10, rtol=1e-10).astype(int)
			polnew = polnew / polnew.sum(axis=1, keepdims=True)

			quit = (pol == polnew).all()
			pol = polnew
			i += 1

		if self._verbose:
			print('N. iterations: ', i)

		return pol, Q

	def trajectory_len(self, x0: str, pol: np.ndarray, traj_len: int, rng_gen: np.random.Generator) -> (np.ndarray, np.ndarray):
		X = self._mdp[0]
		A = self._mdp[1]
		P = self._mdp[2]

		nA = len(A)

		traj = [x0]
		actions = []
		x = list(X).index(x0)

		for _ in range(traj_len):
			a = rng_gen.choice(nA, p=pol[x, :])
			next_states = P[A[a]][x, :]
			x = rng_gen.choice(next_states[:, :, 0], p=next_states[:, :, 1])

			traj += [X[x]]
			actions += [A[a]]

		actions += [A[rng_gen.choice(nA, p=pol[x, :])]]
		return np.array(traj), np.array(actions)

	def trajectory(self, x0: str, pol: np.ndarray, rng_gen: np.random.Generator) -> (np.ndarray, np.ndarray):
		X = self._mdp[0]
		A = self._mdp[1]
		P = self._mdp[2]

		nX = len(X)
		nA = len(A)

		traj = [x0]
		actions = []
		x = list(X).index(x0)
		stop = False
		i = 0

		while not stop:
			a = rng_gen.choice(nA, p=pol[x, :])
			nxt_states = P[A[a]][x, :]
			x = int(rng_gen.choice(nxt_states[:, 0], p=nxt_states[:, 1]))

			traj += [X[x]]
			actions += [A[a]]

			stop = (x in self._goal_states or i > 500)
			if stop:
				actions += [A[rng_gen.choice(nA, p=pol[x, :])]]

			i += 1

		return np.array(traj), np.array(actions)

	def all_trajectories(self, x0: str, pol: np.ndarray, rng_gen: np.random.Generator) -> (np.ndarray, np.ndarray):
		X = self._mdp[0]
		A = self._mdp[1]
		P = self._mdp[2]

		nX = len(X)
		nA = len(A)

		i = 0
		trajs = []
		acts = []
		started_trajs = [[[x0], []]]
		stop = False

		while not stop:
			traj = started_trajs[i][0]
			a_traj = started_trajs[i][1]
			x = list(X).index(traj[-1])
			stop_inner = False
			it = 0
			add_traj = False

			while not stop_inner:
				if x in self._goal_states:
					a_traj += [A[rng_gen.choice(nA, p=pol[x, :])]]
					add_traj = True
					break

				else:
					pol_act = np.nonzero(pol[x, :])[0]
					if len(pol_act) > 1:
						for j in range(1, len(pol_act)):
							if len(np.nonzero(P[A[pol_act[j]]][x, :])) > 1:
								next_states = P[A[pol_act[j]]][x, :]
								x_tmp = rng_gen.choice(next_states[:, :, 0], p=next_states[:, :, 1])
								while x_tmp == x:
									x_tmp = rng_gen.choice(nX, p=P[A[pol_act[j]]][x, :])
							else:
								next_states = P[A[pol_act[j]]][x, :]
								x_tmp = rng_gen.choice(next_states[:, :, 0], p=next_states[:, :, 1])
							tmp_traj = [list(traj) + [X[x_tmp]], list(a_traj) + [A[pol_act[j]]]]
							if tmp_traj not in started_trajs:
								started_trajs += [tmp_traj]

					a = pol_act[0]
					x = rng_gen.choice(nX, p=P[A[a]][x, :])

					if X[x] != traj[-1]:
						traj += [X[x]]
						a_traj += [A[a]]

						stop_inner = (x in self._goal_states)
						if stop_inner:
							a_traj += [A[rng_gen.choice(nA, p=pol[x, :])]]

						add_traj = True

					else:
						if it > 500:
							stop_inner = True

					it += 1

			i += 1
			stop = (i >= len(started_trajs) or i > 500)
			if add_traj:
				trajs += [np.array(traj)]
				acts += [np.array(a_traj)]

		return np.array(trajs, dtype=object), np.array(acts, dtype=object)

	def trajectory_reward(self, trajs: np.ndarray, task_idx: int = None) -> float:
		r_avg = 0
		X = list(self._mdp[0])
		A = list(self._mdp[1])
		if task_idx is not None:
			c = self._mdp[3][task_idx]
		else:
			c = self._mdp[3]
		gamma = self._mdp[4]
		n_trajs = len(trajs)
		t_idx = 1

		for traj in trajs:
			states = traj[0]
			actions = traj[1]
			r_traj = 0
			g = 1

			for idx in range(len(actions)):
				x = states[idx]
				a = actions[idx]
				r_traj += g * c[X.index(x), A.index(a)]
				g *= gamma

			r_avg += r_traj / n_trajs
			if self._verbose:
				print('MDP trajectory reward! Trajectory nr.%d. %.2f%% done' % (t_idx, t_idx / n_trajs * 100), end='\r')
				t_idx += 1

		if self._verbose:
			print('MDP trajectory reward finished!', end='\n')

		return r_avg

class HighDimLegibleMDP(HighDimMDP):

	def __init__(self, x: np.ndarray, a: List[str], p: Dict[str, np.ndarray], gamma: float, verbose: bool, task: str, task_states: List[Tuple[int, int, str]],
				 tasks: List[str], beta: float, goal_states: List[int], sign: int, q_mdps: Dict[str, np.ndarray]):
		self._task = task
		self._tasks = tasks
		self._task_states = {}
		for task_state in task_states:
			self._task_states[task_state[2]] = tuple([task_state[0], task_state[1]])
		self._tasks_q = q_mdps
		self._beta = beta
		nX = len(x)
		nA = len(a)
		nT = len(tasks)
		c = np.zeros((nT, nX, nA))

		super().__init__(x, a, p, c, gamma, goal_states, 'rewards', verbose)
		for t in range(nT):
			for i in range(nX):
				for j in range(nA):
					c[t, i, j] = self.optimal_legible_cost(i, j, t, sign)

		self._mdp = (x, a, p, c, gamma)

	def update_cost_function(self, sign: int):
		mdp = self.mdp
		x = mdp[0]
		a = mdp[1]
		p = mdp[2]
		gamma = mdp[4]
		nX = len(x)
		nA = len(a)
		nT = len(self._tasks)
		c = np.zeros((nT, nX, nA))

		for t in range(nT):
			for i in range(nX):
				for j in range(nA):
					c[t, i, j] = self.optimal_legible_cost(i, j, t, sign)

		self._mdp = (x, a, p, c, gamma)

	def optimal_legible_cost(self, x: int, a: int, t: int, sign: int) -> float:
		task_cost = np.exp(sign * self._beta * self._tasks_q[self._tasks[t]][x, a])
		tasks_sum = task_cost
		for task in self._tasks:
			if task != self._tasks[t]:
				tasks_sum += np.exp(sign * self._beta * self._tasks_q[task][x, a])

		return task_cost / tasks_sum

	def legible_cost(self, x: int, a: int, t: int, sign: int, dist: np.ndarray) -> float:
		task_cost = np.exp(sign * self._beta * self._tasks_q[self._tasks[t]][x, a]) / (1 / dist[t, x])
		tasks_sum = task_cost
		for task in self._tasks:
			if task != self._tasks[t]:
				tasks_sum += np.exp(sign * self._beta * self._tasks_q[task][x, a]) / (1 / dist[self._tasks.index(task), x])

		return task_cost / tasks_sum


import argparse
from inspect import getmembers, isfunction

import numpy

from deepsparse import compile_model, cpu
from deepsparse.utils import verify_outputs
from sparsezoo.models import classification
from sparsezoo.objects import Model

CORES_PER_SOCKET, AVX_TYPE, _ = cpu.cpu_details()

model_registry = dict(getmembers(classification, isfunction))

def fetch_model(model_name: str) -> Model:
    if model_name not in model_registry:
        raise Exception(
            f"Could not find model '{model_name}' in classification model registry."
        )
    return model_registry[model_name]()

def parse_args():
    parser = argparse.ArgumentParser(
        description="Benchmark and check accuracy of image classification models"
    )

    parser.add_argument(
        "model_name",
        type=str,
        choices=model_registry.keys(),
        help="Model type to analyze",
    )

    parser.add_argument(
        "-s",
        "--batch_size",
        type=int,
        default=64,
        help="The batch size to run the analysis for",
    )
    parser.add_argument(
        "-j",
        "--num_cores",
        type=int,
        default=CORES_PER_SOCKET,
        help=(
            "The number of physical cores to run the analysis on, "
            "defaults to all physical cores available on the system"
        ),
    )

    return parser.parse_args()

def calculate_top1_accuracy(pred: numpy.array, labels: numpy.array) -> float:
    """
    :param pred: the model's prediction to compare with
    :param labels: the labels for the data to compare to
    :return: the calculated top1 accuracy
    """
    batch_size = pred.shape[0]
    pred = numpy.argmax(pred, axis=-1)
    labels = numpy.argmax(labels, axis=-1)

    correct = (pred == labels).sum()
    correct *= 100.0 / batch_size

    return correct

def main():
    args = parse_args()
    model = fetch_model(args.model_name)
    batch_size = args.batch_size
    num_cores = args.num_cores

    # Gather batch of data
    batch = model.sample_batch(batch_size=batch_size)
    batched_inputs = batch["inputs"]
    batched_outputs = batch["outputs"]

    # Compile model for inference
    print("Compiling {} model with DeepSparse Engine".format(model.architecture_id))
    engine = compile_model(model, batch_size, num_cores)
    print(engine)

    # INFERENCE
    # Record output from inference through the DeepSparse Engine
    print("Executing...")
    predicted_outputs = engine(batched_inputs)

    # Compare against reference model output
    verify_outputs(predicted_outputs, batched_outputs)

    if "labels" in batch:
        batched_labels = batch["labels"]
        # Measure accuracy against ground truth labels
        accuracy = calculate_top1_accuracy(predicted_outputs[-1], batched_labels[0])
        print("Top-1 Accuracy for batch size {}: {:.2f}%".format(batch_size, accuracy))

    # BENCHMARK
    # Record output from executing through the DeepSparse engine
    print("Benchmarking...")
    results = engine.benchmark(batched_inputs)
    print(results)

if __name__ == "__main__":
    main()


#!/usr/bin/env python3
# _*_ coding: utf-8 _*_

from datetime import datetime
from elasticsearch import Elasticsearch
import pprint

pp = pprint.PrettyPrinter(width=41, compact=True)

def main():
    es = Elasticsearch(
        ['https://3.231.208.103:64297/es/'],
        http_auth=('webuser', 'LINUX@4ever'),
        verify_certs=False,
    )

    res = es.search(
        index="logstash*",
        body={
            "size": 100,
            "query": {
                "range": {
                    "@timestamp": {
                        "gte": "now-1h"
                    }
                }
            }
        },
        scroll = '1m'
    )

    hits = res['hits']['hits']
    scroll_id = res['_scroll_id']
    doc_count = res['hits']['total']['value']

    while doc_count > 0:
        secEvents = []
        for event in hits:
            try:
                secEvent = {
                    'src_ip': event['_source']['src_ip'],
                    'dest_ip': event['_source']['dest_ip'],
                    'dest_port': event['_source']['dest_port']
                }
                if secEvent not in secEvents:
                    secEvents.append(secEvent)
            except:
                continue
        doc_count = doc_count - 100
        res = es.scroll(
            scroll_id=scroll_id,
            scroll = '1m'
        )
        hits = res['hits']['hits']
        scroll_id = res['_scroll_id']

    pp.pprint(secEvents)

if __name__ == "__main__":
    main()


import time
import requests
from fastapi import Form, Depends, File
from fastapi import APIRouter, UploadFile, HTTPException
from sqlalchemy.orm import Session

from core.config import settings
from db.session import get_db
from db.repository.csam_ratio import get_db_data
from schemas.ng import ShowNG

from apis.inspect.base import inspect

router = APIRouter()

@router.post("/upload_file", response_model=ShowNG)
def predict_NG_chips(
    file: UploadFile = File(...), lot_no: str = Form(...), db: Session = Depends(get_db)
):
    print("Received file to process, please wait...")
    start = time.time()
    # Check if lot number exists or if its for testing
    if lot_no.lower()[:4] == "test":
        chip_type = settings.CHIPTYPE
    else:
        prass = requests.get(settings.PRASS_URL + lot_no).json()
        if prass[settings.LOT_NO_COL] == None:
            raise HTTPException(
                status_code=404, detail=f"Lot number: {lot_no} not found in database"
            )
        chip_type = prass[settings.CHIP_TYPE_COL]
    try:
        chips_dict, save_dir, img_shape, no_of_batches, no_of_chips = inspect(
            file, lot_no, db
        )
    except:
        raise HTTPException(
            status_code=400, detail=f"Error while processing uploaded file"
        )

    res = ShowNG(
        plate_no=file.filename.split(".")[0],
        chips=chips_dict,
        img_shape=img_shape,
        no_of_chips=no_of_chips,
        no_of_batches=no_of_batches,
        directory=save_dir,
        chip_type=chip_type,
    )

    print(f"Total Time taken: {round(time.time()-start,2)}")

    return res

@router.get("/read_db")
def read_db(db: Session = Depends(get_db)):
    ratio = get_db_data(db=db)
    return ratio


# -*- coding: utf-8 -*-
"""
Created on Wed Sep  7 09:05:37 2022

@author: Lala
"""
# TALLER 3

import numpy as np
import sympy as sym

# Ejercicios raíces

# 3)

def f(x):
    return 3*x**5 + 5*x**4 - x**3

def deriv(f,x,h=1e-6):
    return (f(x+h)-f(x-h))/(2*h)

def NewtonMethod(f,df,xn,itmax=1000,precision=1e-5):
    error = 1
    it = 0

    while error > precision and it < itmax:
        try:
            xn1 = xn - f(xn)/df(f,xn)
            error = np.abs(f(xn)/df(f,xn))

        except ZeroDivisionError:
            print('Division entre cero')

        xn = xn1
        it += 1

    if it == itmax:
        return False
    else:
        return xn

def GetAllRoots(x,tolerancia=6):
    roots = np.array([])

    for i in x:
        root = NewtonMethod(f, deriv, i)

        if root != False:
            croot = np.round(root,tolerancia)

            if croot not in roots:
                roots = np.append(roots,croot)

    roots.sort()
    return roots

xtrial = np.linspace(-1000,1000,1000)
roots = GetAllRoots(xtrial)
print("raíces:", roots, "\n")

# 4)

def GetLegendre(n):
    x = sym.Symbol('x',Real=True)
    y = sym.Symbol('y',Real=True)

    y = (x**2-1)**n

    p = sym.diff(y,x,n)/(2**n * np.math.factorial(n))
    return p

n=20
Legendre = []

for i in range(n):
    Legendre.append(GetLegendre(i))

def GetAllRootsLegendre(X,tolerancia=6):
        roots = np.array([])

        for i in X:
            for i in range(n):
                root = NewtonMethod(Legendre[n], deriv, i)

            if root != False:
                croot = np.round(root,tolerancia)

                if croot not in roots:
                    roots = np.append(roots,croot)

        roots.sort()
        return roots

roots_Legendre = []
for i in range(n):
    root_Legendre = GetAllRootsLegendre(xtrial)
    roots_Legendre.append(root_Legendre)

# 5)

def GetLaguerre(n):
    x = sym.Symbol('x',Real=True)
    y = sym.Symbol('y',Real=True)

    y = sym.exp(-x)*x**n

    p = sym.exp(x)*sym.diff(y,x,n)/(np.math.factorial(n))
    return p

Laguerre = []

for i in range(n):
    Laguerre.append(GetLaguerre(i))

def GetAllRootsLaguerre(X,tolerancia=6):
        roots = np.array([])

        for i in X:
            for i in range(n):
                root = NewtonMethod(Laguerre[n], deriv, i)

            if root != False:
                croot = np.round(root,tolerancia)

                if croot not in roots:
                    roots = np.append(roots,croot)

        roots.sort()
        return roots

roots_Laguerre = []
for i in range(n):
    root_Laguerre = GetAllRootsLaguerre(xtrial)
    roots_Laguerre.append(root_Laguerre)


#  Shop.py was created by Redjumpman for Redbot
#  This will create a data folder with 3 JSON files and 1 logger
#  The logger will contain information for admin use
import uuid
import discord
import os
import logging
import time
from operator import itemgetter
from discord.ext import commands
from .utils.dataIO import fileIO
from __main__ import send_cmd_help
from .utils import checks
from datetime import datetime
try:   # Check if Tabulate is installed
    from tabulate import tabulate
    tabulateAvailable = True
except:
    tabulateAvailable = False

class Shop:
    """Allows you to purchase items created by the Admins with your credits"""
    # We have to define all the different files were going to load and save
    def __init__(self, bot):
        self.bot = bot
        self.players = fileIO("data/shop/players.json", "load")
        self.shopsys = fileIO("data/shop/shop.json", "load")
        self.pending = fileIO("data/shop/pending.json", "load")
        self.config = fileIO("data/shop/config.json", "load")

    @commands.command(pass_context=True, no_pm=True)
    async def inventory(self, ctx):
        """Shows a list of items you have purchased"""
        user = ctx.message.author
        if user.id in self.players:
            if not self.players[user.id]["Inventory"]:
                await self.bot.say("You have not purchased any items for me to display")
            else:
                    column1 = [subdict["Item Name"] for subdict in self.players[user.id]["Inventory"].values()]
                    column2 = [subdict["Item Quantity"] for subdict in self.players[user.id]["Inventory"].values()]
                    m = list(zip(column1, column2))
                    m.sort()
                    t = tabulate(m, headers=["Item Name", "Item Quantity"])
                    header = "```"
                    header += self.bordered("I N V E N T O R Y")
                    header += "```"
                    if self.config["Inventory Output"] == "Whisper":
                        await self.bot.whisper(header + "```\n" + t + "```")
                    elif self.config["Inventory Output"] == "Chat":
                        await self.bot.say(header + "```\n" + t + "```")
                    else:
                        await self.bot.whisper(header + "```\n" + t + "```")
        else:
            await self.bot.say("You need to join the shop. Use the `beemoshop join` command.")

    @commands.group(pass_context=True, no_pm=True)
    async def shop(self, ctx):
        """Shop Commands. Use !help Shop for other command groups"""

        if ctx.invoked_subcommand is None:
            await send_cmd_help(ctx)

    # We want to seperate the store so it doesn't have a shop prefix
    @shop.command(name="list", pass_context=True)
    async def _list_shop(self, ctx):
        """Shows a list of items that can be purchased"""
        shop_name = self.config["Shop Name"]
        column1 = [subdict["Item Name"] for subdict in self.shopsys.values()]
        column2 = [subdict["Item Cost"] for subdict in self.shopsys.values()]
        print(str(column1))
        if not column1:
            await self.bot.say("There are no items for sale in the shop.")
        else:
            m = list(zip(column1, column2))
            if self.config["Sort Method"] == "Alphabet":
                m = sorted(m)
            elif self.config["Sort Method"] == "Lowest":
                m = sorted(m, key=itemgetter(1), reverse=True)
            elif self.config["Sort Method"] == "Highest":
                m = sorted(m, key=itemgetter(1))
            t = tabulate(m, headers=["Item Name", "Item Cost"])
            header = "```"
            header += self.bordered(shop_name + " Store Listings")
            header += "```"
            if len(t) > 2000:
                first_msg1, first_msg2 = column1[::2], column1[1::2]
                second_msg1, second_msg2 = column2[::2], column2[1::2]
                m1 = list(zip(first_msg1, second_msg1))
                m2 = list(zip(first_msg2, second_msg2))
                if self.config["Sort Method"] == "Alphabet":
                    sorted(m1)
                    sorted(m2)
                elif self.config["Sort Method"] == "Lowest":
                    m1 = sorted(m1, key=itemgetter(1))
                    m2 = sorted(m2, key=itemgetter(1))
                elif self.config["Sort Method"] == "Highest":
                    m1 = sorted(m1, key=itemgetter(1), reverse=True)
                    m2 = sorted(m2, key=itemgetter(1), reverse=True)
                t1 = tabulate(m1, headers=["Item Name", "Item Cost"])
                t2 = tabulate(m2, headers=["Item Name", "Item Cost"])
                if self.config["Store Output"] == "Whisper":
                    await self.bot.whisper(header + "```\n" + t1 + "```")
                    await self.bot.whisper("```" + t2 + "```")
                elif self.config["Store Output"] == "Chat":
                    await self.bot.say(header + "```\n" + t1 + "```")
                    await self.bot.say("```" + t2 + "```")
                else:
                    await self.bot.whisper(header + "```\n" + t1 + "```")
                    await self.bot.whisper("```" + t2 + "```")
            else:
                if self.config["Store Output"] == "Whisper":
                    await self.bot.whisper(header + "```\n" + t + "```")
                elif self.config["Store Output"] == "Chat":
                    await self.bot.say(header + "```\n" + t + "```")
                else:
                    await self.bot.whisper(header + "```\n" + t + "```")

    @shop.command(name="add", pass_context=True, no_pm=True)
    @checks.admin_or_permissions(manage_server=True)
    async def _add_shop(self, ctx, cost: int, *, itemname):
        """Adds an item to the shop list. Max 100 items"""
        if self.config["Shop Items"] < 100:
            self.config["Shop Items"] += 1
            fileIO("data/shop/config.json", "save", self.config)
            shop_name = self.config["Shop Name"]
            self.shopsys[itemname] = {"Item Name": itemname, "Item Cost": cost}
            fileIO("data/shop/shop.json", "save", self.shopsys)
            item_count = len(list(self.shopsys.keys()))
            await self.bot.say("```{} has been added to {} shop for purchase.\nThere is now {} items for sale in the store.```".format(itemname, shop_name, str(item_count)))
        else:
            await self.bot.say("You can only have 100 items for sale in the store.\nDelete an item to add more.")

    @shop.command(name="remove", pass_context=True, no_pm=True)
    @checks.admin_or_permissions(manage_server=True)
    async def _remove_shop(self, ctx, *, itemname):
        """Removes an item from the shop list"""
        shop_name = self.config["Shop Name"]
        if itemname in self.shopsys:
            self.config["Shop Items"] -= 1
            fileIO("data/shop/config.json", "save", self.config)
            del self.shopsys[itemname]
            fileIO("data/shop/shop.json", "save", self.shopsys)
            await self.bot.say("```{} has been removed from {} shop.```".format(itemname, shop_name))
        else:
            await self.bot.say("That item is not in {}'s store listings".format(shop_name))

    @shop.command(name="redeem", pass_context=True, no_pm=True)
    async def _redeem_shop(self, ctx, *, itemname):
        """Sends a request to redeem an item"""
        user = ctx.message.author
        time_now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        role = self.config["Shop Role"]
        if self.inventory_item_check(user.id, itemname):
            confirmation_number = str(uuid.uuid4())
            self.pending[confirmation_number] = {"Confirmation Number": confirmation_number,
                                                 "Time Stamp": time_now,
                                                 "Name": user.name,
                                                 "ID": user.id,
                                                 "Item": itemname}
            fileIO("data/shop/pending.json", "save", self.pending)
            self.inventory_remove(user.id, itemname)
            if self.config["Shop Notify"]:
                names = self.role_check(role, ctx)
                destinations = [m for m in ctx.message.server.members if m.name in names]
                for destination in destinations:
                    await self.bot.send_message(destination, "{} was added to the pending list by {}.".format(itemname, user.name))
                await self.bot.say("```{} has been added to pending list. Your confirmation number is {}.```".format(itemname, confirmation_number))
            await self.bot.say("```{} has been added to pending list. Your confirmation number is {}.```".format(itemname, confirmation_number))
        else:
            await self.bot.say("You do not have that item to redeem")

    @shop.command(name="buy", pass_context=True, no_pm=True)
    async def _buy_shop(self, ctx, *, itemname):
        """Buy an item from the store list"""
        shop_name = self.config["Shop Name"]
        user = ctx.message.author
        if self.config["Shop Open"]:
            if self.inventory_check(user.id):
                if itemname in self.shopsys:
                    points = self.shopsys[itemname]["Item Cost"]
                    if self.account_check(user):
                        if self.enough_points(user, points):
                            points = self.shopsys[itemname]["Item Cost"]
                            self.inventory_add(user.id, itemname)
                            bank = self.bot.get_cog("Economy").bank
                            bank.withdraw_credits(user, points)
                            await self.bot.say("```You have purchased a {} for {} credits.\n{} has been added to your inventory.```".format(itemname, str(points), itemname))
                        else:
                            await self.bot.say("You don't have enough credits to purchase this item.")
                    else:
                        await self.bot.say("You do not have a bank account.")
                else:
                    await self.bot.say("This item is not in the shop.")
            else:
                await self.bot.say("You need to join the {} shop to purchase items. Example: beemoshop join".format(shop_name))
        else:
            await self.bot.say("{} shop is currently closed.".format(shop_name))

    @shop.command(name="gift", pass_context=True, no_pm=True)
    async def _gift_shop(self, ctx, user: discord.Member, *, itemname):
        """Send a gift in your inventory to another member"""
        author = ctx.message.author
        if author == user:
            await self.bot.say("You can't give an item to yourself.")
            return
        if len(itemname) < 0:
            await self.bot.say("You need to tell me what you want to transfer.")
            return
        if user.id in self.players:
            if self.inventory_item_check(author.id, itemname):
                if self.inventory_item_amount(author.id, itemname):
                    nametwo = self.shopsys[itemname]["Item Name"]
                    self.inventory_remove(author.id, itemname)
                    self.inventory_add(user.id, itemname)
                    logger.info("{}({}) gifted a {} item to {}({})".format(author.name, author.id, itemname, user.name, user.id))
                    await self.bot.say("```I have gifted {} to {}'s inventory```".format(nametwo, user.name))
                else:
                    await self.bot.say("You currently don't own any of these.")
            else:
                await self.bot.say("You do not own this shop item.")
        else:
            shop_name = self.config["Shop Name"]
            await self.bot.say("I cant find a user with that name." +
                               " Check to see if that user has joined {} shop. They need to type beemoshop join before they can recieve a gift.".format(shop_name))

    @shop.command(name="trade", pass_context=True, no_pm=True)
    async def _trade_shop(self, ctx, user: discord.Member, *, tradeoffer: str):
        """Requests a trade with another shop member"""
        author = ctx.message.author
        if author.id in self.players:
            if user.id in self.players:
                if not self.players[user.id]["Block Trades"]:
                    if await self.check_cooldowns(user.id):
                        if tradeoffer in self.players[author.id]["Inventory"]:
                            await self.bot.say("{} requests a trade with {}. Do you wish to trade for {}?".format(author.mention, user.mention, tradeoffer))
                            answer = await self.bot.wait_for_message(timeout=15, author=user)
                            if not answer:
                                await self.bot.say("No response. Cancelling trade with {}.".format(user.name))
                            elif answer.content.title() == "No":
                                await self.bot.say("{} has rejected your trade.".format(user.name))
                            elif answer.content.title() == "Yes":
                                await self.bot.say("Please say which item you would like to trade.")
                                response = await self.bot.wait_for_message(timeout=15, author=user)
                                if not response:
                                    await self.bot.say("No response. Cancelling trade with {}.".format(user.name))
                                elif response.content.title() in self.players[user.id]["Inventory"]:
                                    await self.bot.say("{} has offered {}, do you wish to accept this trade, {}?".format(user.mention, response.content, author.mention))
                                    reply = await self.bot.wait_for_message(timeout=15, author=author)
                                    if not reply:
                                        await self.bot.say("No response. Cancelling trade with {}.".format(user.name))
                                    elif reply.content.title() == "No" or reply.content.title() == "Cancel":
                                        await self.bot.say("Trade Rejected. Cancelling trade.")
                                    elif reply.content.title() == "Yes" or reply.content.title() == "Accept":
                                        self.inventory_remove(author.id, tradeoffer)
                                        self.inventory_remove(user.id, response.content.title())
                                        self.inventory_add(user.id, tradeoffer)
                                        self.inventory_add(author.id, response.content.title())
                                        await self.bot.say("Trading items... {} recieved {}, and {} recieved {}.".format(author.mention, response.content, user.mention, tradeoffer))
                                        await self.bot.say("Trade complete.")
                                    else:
                                        await self.bot.say("Invalid response. Cancelling trade with {}.".format(user.name))
                                else:
                                    await self.bot.say("You don't have this item. Cancelling trade.")
                            else:
                                await self.bot.say("Invalid response. Cancelling trade with {}.".format(user.name))
                        else:
                            await self.bot.say("You don't have this item. Cancelling trade.")
                else:
                    await self.bot.say("This player is currently blocking trades.")
            else:
                await self.bot.say("This person is not a member of the shop.")
        else:
            await self.bot.say("You need to join the shop before you can trade items.")

    @shop.command(name="join", pass_context=True, no_pm=True)
    async def _join_shop(self, ctx):
        """Adds you to the shop. Only need to do this once."""
        shop_name = self.config["Shop Name"]
        user = ctx.message.author
        if user.id not in self.players:
            self.players[user.id] = {}
            fileIO("data/shop/players.json", "save", self.players)
            self.players[user.id] = {"Inventory": {},
                                     "Block Trades": False,
                                     "Trade Cooldown": 0}
            fileIO("data/shop/players.json", "save", self.players)
            await self.bot.say("```You have joined {} shop. You can now buy items with credits.```".format(shop_name))
        else:
            await self.bot.say("```You have already a member of the {} shop```".format(shop_name))

    @shop.command(name="blocktrades", pass_context=True, no_pm=True)
    async def _blocktrades_shop(self, ctx, cooldown: int):
        """Toggles blocking trade requests on and off."""
        user = ctx.message.author
        if user.id in self.players:
            if self.players[user.id]["Block Trades"] is False:
                self.players[user.id]["Block Trades"] = True
                fileIO("data/shop/config.json", "save", self.config)
                await self.bot.say("Now blocking all trades")
            else:
                self.players[user.id]["Block Trades"] = False
                fileIO("data/shop/config.json", "save", self.config)
                await self.bot.say("Trades are now enabled.")

    @commands.group(pass_context=True, no_pm=True)
    async def setshop(self, ctx):
        """Shop configuration settings"""
        if ctx.invoked_subcommand is None:
            await send_cmd_help(ctx)

    @setshop.command(name="notify", pass_context=True, no_pm=True)
    @checks.admin_or_permissions(manage_server=True)
    async def _notify_setshop(self, ctx):
        """PM's all users with Shopkeeper role. Add this role to be notified.
        This command will toggle notifications on/off"""
        if self.config["Shop Notify"]:
            self.config["Shop Notify"] = not self.config["Shop Notify"]
            fileIO("data/shop/config.json", "save", self.config)
            await self.bot.say("Shop notifications are now OFF!")
        else:
            self.config["Shop Notify"] = True
            fileIO("data/shop/config.json", "save", self.config)
            await self.bot.say("Shop notifications are now ON!")

    @setshop.command(name="output", pass_context=True, no_pm=True)
    @checks.admin_or_permissions(manage_server=True)
    async def _output_setshop(self, ctx, command: str, output: str):
        """Sets the output to chat/whisper for inventory & shop list"""
        command = command.title()
        output = output.title()
        if command == "Shop":
            if output == "Chat":
                self.config["Store Output"] = "Chat"
                fileIO("data/shop/config.json", "save", self.config)
                await self.bot.say("Store listings will now display in chat.")
            elif output == "Whisper" or output == "Pm" or output == "Dm":
                self.config["Store Output"] = "Whisper"
                fileIO("data/shop/config.json", "save", self.config)
                await self.bot.say("Store listings will now display in whisper.")
            else:
                await self.bot.say("Output must be Chat or Whisper/DM/PM.")
        elif command == "Inventory":
            if output == "Chat":
                self.config["Inventory Output"] = "Chat"
                fileIO("data/shop/config.json", "save", self.config)
                await self.bot.say("Inventory will now display in chat.")
            elif output == "Whisper" or output == "Pm" or output == "Dm":
                self.config["Inventory Output"] = "Whisper"
                fileIO("data/shop/config.json", "save", self.config)
                await self.bot.say("Inventory will now display in whisper.")
            else:
                await self.bot.say("Output must be Chat or Whisper/DM/PM.")
        else:
            await self.bot.say("Command must be Shop or Inventory.")

    @setshop.command(name="tradecd", pass_context=True, no_pm=True)
    @checks.admin_or_permissions(manage_server=True)
    async def _tcd_setshop(self, ctx, cooldown: int):
        """Sets the cooldown timer for trading, in seconds."""
        self.config["Trade Cooldown"] = cooldown
        fileIO("data/shop/config.json", "save", self.config)
        await self.bot.say("Trading cooldown set to {}".format(self.time_format(cooldown)))

    @setshop.command(name="role", pass_context=True, no_pm=True)
    @checks.admin_or_permissions(manage_server=True)
    async def _role_sethop(self, ctx, *, rolename: str):
        """Change the name of the notification role"""
        self.config["Shop Role"] = rolename
        fileIO("data/shop/config.json", "save", self.config)
        await self.bot.say("Notify role set to {}. Assign this role to users you want to haven notifed of pending items.".format(rolename))

    @setshop.command(name="toggle", pass_context=True, no_pm=True)
    @checks.admin_or_permissions(manage_server=True)
    async def _toggle_setshop(self, ctx):
        """Opens and closes the shop"""
        shop_name = self.config["Shop Name"]
        if self.config["Shop Open"]:
            self.config["Shop Open"] = not self.config["Shop Open"]
            fileIO("data/shop/config.json", "save", self.config)
            await self.bot.say("The shop is now closed.")
        else:
            self.config["Shop Open"] = True
            fileIO("data/shop/config.json", "save", self.config)
            await self.bot.say("{} shop is now open for business!".format(shop_name))

    @setshop.command(name="sorting", pass_context=True, no_pm=True)
    @checks.admin_or_permissions(manage_server=True)
    async def _sort_setshop(self, ctx, choice: str):
        """Changes the sorting method for shop listings. Alphabet, Lowest, Highest"""
        choice = choice.title()
        if choice == "Alphabet":
            self.config["Sort Method"] = "Alphabet"
            fileIO("data/shop/config.json", "save", self.config)
            await self.bot.say("Changing sorting method to Alphabetical.")
        elif choice == "Lowest":
            self.config["Sort Method"] = "Lowest"
            fileIO("data/shop/config.json", "save", self.config)
            await self.bot.say("Setting sorting method to Lowest.")
        elif choice == "Highest":
            self.config["Sort Method"] = "Highest"
            fileIO("data/shop/config.json", "save", self.config)
            await self.bot.say("Setting sorting method to Highest.")
        else:
            await self.bot.say("Please choose Alphabet, Lowest, or Highest.")

    @setshop.command(name="name", pass_context=True, no_pm=True)
    @checks.admin_or_permissions(manage_server=True)
    async def _name_setshop(self, ctx, *, name):
        """Renames the shop"""
        shop_name = self.config["Shop Name"]
        if len(name) > 0:
            self.config["Shop Name"] = name
            fileIO("data/shop/config.json", "save", self.config)
            shop_name = self.config["Shop Name"]
            await self.bot.say("I have renamed the shop to {}.".format(shop_name))
        else:
            await self.bot.say("You need to enter a name for the shop.")

    @commands.group(name="pending", pass_context=True)
    async def _pending(self, ctx):
        """List of pending commands for redemable items"""
        if ctx.invoked_subcommand is None:
            await send_cmd_help(ctx)

    @_pending.command(pass_context=True, no_pm=True)
    async def check(self, ctx, *, code):
        """Checks if an item is on the pending list with conf #"""
        if code in self.pending:
            item = self.pending[code]["Item"]
            await self.bot.say("{} is still on the pending list.".format(item))
        else:
            await self.bot.say("This code is either not valid, or the item is no longer on the list.")

    @_pending.command(pass_context=True, no_pm=True)
    @checks.admin_or_permissions(manage_server=True)
    async def show(self, ctx):
        """Shows a list of items waiting to be redeemed"""
        if len(self.pending) > 0:
            keys = ["Item", "User Name", "User ID", "Time Stamp", "Confirmation Number"]
            items = []
            user_names = []
            user_ids = []
            time_stamps = []
            conf_nums = []
            for x in self.pending:
                items.append(self.pending[x]["Item"])
                user_names.append(self.pending[x]["Name"])
                time_stamps.append(self.pending[x]["Time Stamp"])
                conf_nums.append(self.pending[x]["Confirmation Number"])
                user_ids.append(self.pending[x]["ID"])
            table = list(zip(items, user_names, user_ids, time_stamps, conf_nums))
            t = tabulate(table, headers=keys, numalign="left")
            await self.bot.say("```" + t + "```")
        else:
            await self.bot.say("The pending list is empty.")

    @_pending.command(pass_context=True, no_pm=True)
    @checks.admin_or_permissions(manage_server=True)
    async def clear(self, ctx, *, number):
        """Allows you to clear one item by confirmation number"""
        if len(self.pending) > 0:
            if number in self.pending:
                name = self.pending[number]["Name"]
                item = self.pending[number]["Item"]
                del self.pending[number]
                fileIO("data/shop/pending.json", "save", self.pending)
                await self.bot.say("{}'s {} has been cleared from the pending list.'.".format(name, item))
            else:
                await self.bot.say("Could not find this code in the pending. It may have been cleared or does not exist.")
        else:
            await self.bot.say("The pending list is empty.")

    @_pending.command(pass_context=True, no_pm=True)
    @checks.admin_or_permissions(manage_server=True)
    async def clearall(self, ctx):
        """Clears all items from the pending list"""
        if len(self.pending) > 0:
            del self.pending
            self.pending = {}
            fileIO("data/shop/pending.json", "save", self.pending)
            await self.bot.say("Pending list now cleared.")
        else:
            await self.bot.say("Nothing in pending list to clear.")

    async def check_cooldowns(self, userid):
        if abs(self.players[userid]["Trade Cooldown"] - int(time.perf_counter())) >= self.config["Trade Cooldown"]:
            self.players[userid]["Trade Cooldown"] = int(time.perf_counter())
            fileIO("data/shop/players.json", "save", self.players)
            return True
        elif self.players[userid]["Trade Cooldown"] == 0:
            self.players[userid]["Trade Cooldown"] = int(time.perf_counter())
            fileIO("data/shop/players.json", "save", self.players)
            return True
        else:
            s = abs(self.players[userid]["Trade Cooldown"] - int(time.perf_counter()))
            seconds = abs(s - self.config["Trade Cooldown"])
            await self.bot.say("You must wait before trading again. You still have: {}".format(self.time_format(seconds)))
            return False

    def bordered(self, text):
        lines = text.splitlines()
        width = max(len(s) for s in lines)
        res = ["┌" + "─" * width + "┐"]
        for s in lines:
            res.append("│" + (s + " " * width)[:width] + "│")
        res.append("└" + "─" * width + "┘")
        return "\n".join(res)

    def time_format(self, seconds):
        m, s = divmod(seconds, 60)
        h, m = divmod(m, 60)
        if h > 0:
            msg = "{} hours, {} minutes, {} seconds".format(h, m, s)
        elif h == 0 and m > 0:
            msg = "{} minutes, {} seconds".format(m, s)
        elif m == 0 and h == 0 and s > 0:
            msg = "{} seconds".format(s)
        elif m == 0 and h == 0 and s == 0:
            msg = "No cooldown"
        return msg

    def account_check(self, uid):
        bank = self.bot.get_cog("Economy").bank
        if bank.account_exists(uid):
            return True
        else:
            return False

    def role_check(self, role, ctx):
        return [m.name for m in ctx.message.server.members if role.lower() in [str(r).lower() for r in m.roles] and str(m.status) != "offline"]

    def enough_points(self, uid, amount):
        bank = self.bot.get_cog("Economy").bank
        if self.account_check(uid):
            if bank.can_spend(uid, amount):
                return True
            else:
                return False

    def inventory_check(self, uid):
        if uid in self.players:
            return True
        else:
            return False

    def inventory_item_check(self, uid, itemname):
        if self.inventory_check(uid):
            if itemname in self.players[uid]["Inventory"]:
                return True
            else:
                return False
        else:
            False

    def inventory_item_amount(self, uid, itemname):
            if self.inventory_check(uid):
                if self.inventory_item_check(uid, itemname):
                    if self.players[uid]["Inventory"][itemname]["Item Quantity"] > 0:
                        return True
                    else:
                        return False
                else:
                    return False
            else:
                return False

    def inventory_add(self, uid, itemname):
        if self.inventory_check(uid):
            if self.inventory_item_check(uid, itemname):
                self.players[uid]["Inventory"][itemname]["Item Quantity"] += 1
                fileIO("data/shop/players.json", "save", self.players)
            else:
                self.players[uid]["Inventory"][itemname] = {"Item Name": itemname,
                                                            "Item Quantity": 1}
                fileIO("data/shop/players.json", "save", self.players)
        else:
            return False

    def inventory_remove(self, uid, itemname):
        if self.inventory_check(uid):
            if self.inventory_item_check(uid, itemname):
                self.players[uid]["Inventory"][itemname]["Item Quantity"] -= 1
                fileIO("data/shop/players.json", "save", self.players)
                if self.players[uid]["Inventory"][itemname]["Item Quantity"] < 1:
                    del self.players[uid]["Inventory"][itemname]
                    fileIO("data/shop/players.json", "save", self.players)
            else:
                return False
        else:
            return False

def check_folders():
    if not os.path.exists("data/shop"):
        print("Creating data/shop folder...")
        os.makedirs("data/shop")

def check_files():
    pconfig = {"Block Trades": False,
               "Trade Cooldown": 0}
    # For people using an old version of shop. Will be removed at a later date NOTE
    try:
        shop_dict = fileIO("data/shop/shop.json", "load")
        shop_item_count = len(list(shop_dict.keys()))
        system = {"Shop Name": "ARDENT",
                  "Shop Open": True,
                  "Shop Notify": False,
                  "Shop Items": shop_item_count,
                  "Shop Role": "Shopkeeper",
                  "Trade Cooldown": 30,
                  "Store Output": "Whisper",
                  "Inventory Output": "Whisper",
                  "Sort Method": "Alphabet"}
    except:
        system = {"Shop Name": "ARDENT",
                  "Shop Open": True,
                  "Shop Notify": False,
                  "Shop Items": 0,
                  "Trade Cooldown": 30,
                  "Store Message": "Whisper",
                  "Inventory Output": "Whisper",
                  "Shop Role": "Shopkeeper",
                  "Sort Method": "Alphabet"}

    f = "data/shop/pending.json"
    if not fileIO(f, "check"):
        print("Creating default shop pending.json...")
        fileIO(f, "save", {})

    f = "data/shop/shop.json"
    if not fileIO(f, "check"):
        print("Creating default shop shop.json...")
        fileIO(f, "save", {})

    f = "data/shop/players.json"
    if not fileIO(f, "check"):
        print("Adding shop player.json...")
        fileIO(f, "save", {})
    else:
        current = fileIO(f, "load")
        if current.keys() is not None:
            for player in current:
                    for key in pconfig.keys():
                        if key not in current[player].keys():
                            current[player][key] = pconfig[key]
                            print("Adding " + str(key) + " field to shop players.json")
                            fileIO(f, "save", current)
    f = "data/shop/config.json"
    if not fileIO(f, "check"):
        print("Adding shop config.json...")
        fileIO(f, "save", system)
    else:  # consistency check
        current = fileIO(f, "load")
        if current.keys() != system.keys():
            for key in system.keys():
                if key not in current.keys():
                    current[key] = system[key]
                    print("Adding " + str(key) +
                          " field to shop config.json")
            fileIO(f, "save", current)

def setup(bot):
    global logger
    check_folders()
    check_files()
    logger = logging.getLogger("shop")
    if logger.level == 0:  # Prevents the logger from being loaded again in case of module reload
        logger.setLevel(logging.INFO)
        handler = logging.FileHandler(filename="data/shop/shop.log", encoding="utf-8", mode="a")
        handler.setFormatter(logging.Formatter("%(asctime)s %(message)s", datefmt="[%d/%m/%Y %H:%M]"))
        logger.addHandler(handler)
    if tabulateAvailable:
        bot.add_cog(Shop(bot))
    else:
        raise RuntimeError("You need to run 'pip3 install tabulate'")


import bs4
import requests
from requests import status_codes
import pandas as pd
import textwrap
from bs4 import BeautifulSoup as soup

papers = pd.read_csv('dataset.csv')
url = pd.DataFrame(papers, columns=['GTRProjectUrl', 'FundingOrgName'])
epsrc = url.loc[url['FundingOrgName'] == "EPSRC"]
abstracts = epsrc['GTRProjectUrl'].values

filename = "abstract1.csv"
f = open(filename, "w")
header = "All Abstracts"
f.write(header)

for abstract in abstracts:
    result = requests.get(abstract)
    if result.status_code == requests.codes.ok:
        src = result.content
        extract = soup(src, 'html.parser')
        content = extract.find("gtr:abstracttext")
        title = extract.find("gtr:title")
        funding = extract.find("gtr:valuepounds")
        print("Project Title: " + title.string)
        print("Funding Organization: EPSRC")
        print("Project Abstract: " + textwrap.fill(content.string), end='\n')
        print("Funded Value : £" + funding.string)
        print(len(abstracts))
        print(abstract)

        f.write("\n" + content.string.replace(",", "|"))

f.close()


import matplotlib.pyplot as plt;
import pandas as pd
import numpy as np
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier , AdaBoostClassifier
from sklearn import tree
from sklearn.ensemble import VotingClassifier
from sklearn import preprocessing
from sklearn import metrics
from sklearn.metrics import accuracy_score

trainSize = 400000
testSize = 50000
size = trainSize+testSize # 400000 for taining and 50000 for testing
data = pd.read_csv("mixed.csv", nrows=size)
data = data.drop(columns="No.")
columns_titles = ["Time", "Source", "Destination", "Length", "Info", "Protocol"]
data = data.reindex(columns=columns_titles)
label_names = np.unique(data.iloc[:,5])

le = preprocessing.LabelEncoder()
data.iloc[:,1] = le.fit_transform(data.iloc[:,1])
data.iloc[:,2] = le.fit_transform(data.iloc[:,2])
data.iloc[:,4] = le.fit_transform(data.iloc[:,4])
data.iloc[:,5] = le.fit_transform(data.iloc[:,5])

def chooseObsMan(dic_f):
    chosenIDs=np.array(())
    for k,d in dic_f.items():
        temp=np.where(y_train==k)[0]
        if temp.size < dic_f[k]:
            temp = dic_f[k]
        z=np.random.choice(temp,dic_f[k],replace=False)
        chosenIDs=np.concatenate((chosenIDs,z))
    return np.int64(chosenIDs)
def label_index(row,threshold=0):

    if threshold==0:
        threshold=np.max(row)
    temp_i=np.where(row>=threshold)[0]
    if temp_i.shape[0]>0:
        return temp_i[0]+1
    else:
        return -1

y_train = data.iloc[:trainSize,5]
x_train = data.iloc[:trainSize,:-1]
#x_train = preprocessing.normalize(data.iloc[:trainSize,:-1])

y_test = data.iloc[trainSize:,5]
x_test = data.iloc[trainSize:,:-1]
#x_test = preprocessing.normalize(data.iloc[trainSize:,:-1])

li = {}
r = int(size/100000 + 1)*30
for i in np.unique(data.iloc[:,-1]):
    if data['Protocol'].value_counts()[i] < r:
        li.update({i : data['Protocol'].value_counts()[i]})
    else :
        li.update({i : r})

labelled=np.zeros(x_train.shape[0])
labelled[:]=-1
labelledIndex=chooseObsMan(li)
labelled[labelledIndex]=y_train[labelledIndex]

#---------------------------------------------OurModel----------------------------------------
clfs ={}

#clf1=GaussianNB()
#clf2=KNeighborsClassifier(n_jobs=-1)
clf3=RandomForestClassifier ( n_estimators = 20 , random_state = 0 ,n_jobs=-1)
#clf4=SVC(kernel='rbf', probability=True)
clf5=tree.DecisionTreeClassifier()
clf7=AdaBoostClassifier()
#clf8=QuadraticDiscriminantAnalysis()

clfs1=[('AdaBoost', clf7),('j48', clf5), ('RFC', clf3)]
ensemble_models = VotingClassifier(estimators=clfs1, voting='soft')

threshold=0.3
leftNumber=-1
temp_left_Group=np.arange(x_train.shape[0])
for i in range(0,x_train.shape[0]):
    temp_left=sum(labelled==-1)
    #print(i, temp_left)
    if temp_left<=0:
        break
    #ensemble_models = VotingClassifier(estimators=clfs1, voting='soft')
    ensemble_models.fit(x_train[labelled!=-1],y_train[labelled!=-1])
    predicted=ensemble_models.predict_proba(x_train[labelled==-1])
    final_decition=np.apply_along_axis(label_index, 1, predicted,threshold)
    labelled[labelled==-1]=final_decition
    if temp_left==leftNumber:
        threshold-0.01
    threshold_step=0.01
    leftNumber=temp_left
    temp_left=sum(labelled==-1)
    temp_left_Group[i]=temp_left
    #print(temp_left, threshold, (final_decition != -1).mean() *400000 , (predicted != -1).mean() *400000)
    if i >0:
        change_rate=temp_left_Group[i-1] - temp_left_Group[i]
        #print(change_rate)
        if(change_rate<1000):
            threshold=threshold-threshold_step

print("------------------------------------Our Module---------------------------------------")
print('threshold step=',threshold_step)

print('\n*********Training*********\n')

pre=labelled[labelled!=-1]
org=y_train[labelled!=-1]
pop=metrics.precision_recall_fscore_support(org, pre)
results = pd.DataFrame()
for i in range(0,len(pop)):
    if len(pop[0]) > np.unique(data.iloc[:,5]).shape[0]:
        temp = pd.DataFrame([pop[i][:-1]],columns=label_names)
    else :
        temp = pd.DataFrame([pop[i]],columns=label_names)

    results=results.append(temp, ignore_index=True)
#print(len(clfs1),'Classifier')

results.insert(0,'  ',['Precision','Recall','Fscore','Support'])
print(results)
resultsDataFrame1 = results
correctLabelled=metrics.accuracy_score(org, pre,normalize=False)
incorrectLabelled=sum(results.iloc[3,1:])-correctLabelled
print([correctLabelled,incorrectLabelled])
numbers=results.iloc[3,1:]
correctLabelledForEachclass=results.iloc[3,1:] *results.iloc[1,1:]
incorrectLabelledForEachclass=results.iloc[3,1:]-correctLabelledForEachclass
results1 = pd.DataFrame(columns = ['Total','Correct', 'Incorrect'])
results1['Total'] = numbers
results1['Correct'] = correctLabelledForEachclass
results1['Incorrect'] = incorrectLabelledForEachclass
print(results1)


import logging

import torch
from torch import nn
import torch.nn.functional as F
from torchmetrics.functional.classification.accuracy import accuracy
import pytorch_lightning as pl
import numpy as np

from .warmup_scheduler import WarmupScheduler

class AbstractBaseClassifierModel(pl.LightningModule):
    """Abstract class classifiers.
    Reusable code for clasifier models.
    To make new classifiers, just implement the initializer and the forward method
    """

    def __init__(self,
                 optimizer=None, optimizer_kwargs={},
                 scheduler=None, scheduler_kwargs={},
                 param_scheduler=None, param_scheduler_kwargs={},
                 warmup_epochs=0, eval_mode='class_accuracy',
                 shift_seed=7,shift_max=None,
                 shift_samples=None,shift_patch_size=None
    ):
        super().__init__()
        self.optimizer_fn = optimizer if optimizer is not None else torch.optim.Adam
        self.optimizer_kwargs = optimizer_kwargs
        self.scheduler_fn = scheduler
        self.scheduler_kwargs = scheduler_kwargs
        self.param_scheduler_fn = param_scheduler
        self.param_scheduler_kwargs = param_scheduler_kwargs
        self.warmup_epochs = warmup_epochs

        # Evaluation settings
        self.eval_mode = eval_mode
        self.shift_seed = shift_seed
        self.shift_max = shift_max
        self.shift_samples = shift_samples
        self.shift_patch_size = shift_patch_size
        logging.info(f'Evaluation mode: {self.eval_mode}')

    # logic for a single training step
    def training_step(self, batch, batch_idx):
        x, y = batch
        logits = self(x)
        loss = F.nll_loss(logits, y)

        # training metrics
        preds = torch.argmax(logits, dim=1)
        acc = accuracy(preds, y)
        self.log('train_loss', loss, on_step=True, on_epoch=True, logger=True)
        self.log('train_acc', acc, on_step=True, on_epoch=True, logger=True)
        return loss

    # logic for a single validation step
    def validation_step(self, batch, batch_idx):
        x, y = batch
        logits = self(x)
        loss = F.nll_loss(logits, y)

        # validation metrics
        preds = torch.argmax(logits, dim=1)
        acc = accuracy(preds, y)
        self.log('val_loss',loss,prog_bar=True,sync_dist=True)
        self.log('val_acc',acc,prog_bar=True,sync_dist=True)
        return loss

    def on_test_start(self) -> None:
        """Called when the test begins."""
        np.random.seed(self.shift_seed)

    # logic for a single testing step
    def test_step(self, batch, batch_idx):
        x, y = batch
        logits = self(x)
        loss = F.nll_loss(logits, y)

        # test metrics
        if self.eval_mode=='class_accuracy':
            preds = torch.argmax(logits,dim=1)
            acc = accuracy(preds,y)
            self.log('test_loss',loss,prog_bar=True,sync_dist=True)
            self.log('test_acc',acc,prog_bar=True,sync_dist=True)
        elif self.eval_mode=='shift_consistency':
            self.eval_consistency_step(batch,batch_idx,'test','shift')
            self.eval_consistency_step(batch,batch_idx,'test','circular')
        return loss

    def eval_consistency_step(self, batch, batch_idx,
                              val_mode='test', mode='shift'):
        assert mode in ['shift', 'circular']
        x,_ = batch

        # Read dataset from 'inc_conv1_support'
        if self.inc_conv1_support: dataset = 'imagenet'
        else: dataset = 'cifar10'

        if dataset=='imagenet':
            outputs = []
            if mode == 'shift':
                # Pass shifted inputs
                offsets = [np.random.randint(self.shift_max,size=2) for j in range(0,self.shift_samples)]
                for j in range(0,self.shift_samples):
                    outputs.append(self(x[:,:,offsets[j][0]:offsets[j][0]+self.shift_patch_size,offsets[j][1]:offsets[j][1]+self.shift_patch_size]))
                # Compute consistency
                cur_agree = self.agreement(outputs,self.shift_samples).type(torch.FloatTensor).to(outputs[0].device)
            elif mode == 'circular':
                # Pass rolled inputs
                # -max to max for comparison purposes
                offsets = [np.random.randint(-self.shift_max,self.shift_max,size=2) for j in range(0,self.shift_samples)]
                for j in range(0,self.shift_samples):
                    outputs.append(self(torch.roll(x,shifts=(offsets[j][0],offsets[j][1]),dims=(2,3))))
                # Compute consistency
                cur_agree = self.agreement(outputs,self.shift_samples).type(torch.FloatTensor).to(outputs[0].device)
        elif dataset=='cifar10':
            max_shift = 3
            random_shift1 = torch.randint(-max_shift, max_shift, (2,))
            random_shift2 = torch.randint(-max_shift, max_shift, (2,))
            if mode == 'shift':
                pad_lengths = (max_shift, max_shift, max_shift, max_shift)
                i1_l, i1_r = random_shift1[0] + max_shift, random_shift1[0]-max_shift
                j1_l, j1_r = random_shift1[1] + max_shift, random_shift1[1]-max_shift
                i2_l, i2_r = random_shift2[0] + max_shift, random_shift2[0]-max_shift
                j2_l, j2_r = random_shift2[1] + max_shift, random_shift2[1]-max_shift
                shifted_x1 = F.pad(x, pad_lengths)[:, :, i1_l:i1_r, j1_l:j1_r ]
                shifted_x2 = F.pad(x, pad_lengths)[:, :, i2_l:i2_r, j2_l:j2_r ]
            elif mode == 'circular':
                shifted_x1 = torch.roll(x, shifts = (random_shift1[0], random_shift1[1]), dims = (2, 3))
                shifted_x2 = torch.roll(x, shifts = (random_shift2[0], random_shift2[1]), dims = (2, 3))

            # Compute consistency
            shifted_preds1 = torch.argmax(self(shifted_x1),1)
            shifted_preds2 = torch.argmax(self(shifted_x2),1)
            cur_agree = accuracy(shifted_preds1,shifted_preds2)
        self.log('%s_%s_consistency' % (val_mode, mode), cur_agree, prog_bar=True)
        return

    # (Core) Compute consistency
    def agreement(self,outputs,robust_num):
        preds = torch.stack([output.argmax(dim=1,keepdim=False) for output in outputs], dim=0)
        similarity = torch.sum((preds == preds[0:1,:]).int(), dim=0)
        agree = 100*torch.mean((similarity == robust_num).float())
        return agree

    def optimizer_step( self,epoch,batch_idx,
                        optimizer,optimizer_idx,optimizer_closure,
                        on_tpu,using_native_amp,using_lbfgs):
        """Called when the train epoch begins."""
        # Lr warmup
        if self.current_epoch < self.warmup_epochs:
            it_curr = self.trainer.num_training_batches*self.current_epoch+1+batch_idx
            it_max = self.trainer.num_training_batches*self.warmup_epochs
            lr_scale = float(it_curr) / it_max
            for pg in self.trainer.optimizers[0].param_groups:
                pg['lr'] = lr_scale * self.learning_rate

        # Update params
        optimizer.step(closure=optimizer_closure)
        optimizer.zero_grad()

    def configure_optimizers(self):
        logging.info(f'Configuring optimizer: {self.optimizer_fn} with {self.optimizer_kwargs}')

        # Filter net/pool parameters
        net_pars, pool_pars = [], []
        for n,p in self.named_parameters():
            if "component_selection" in n and p.requires_grad_:
                pool_pars.append(p)
            elif p.requires_grad_:
                net_pars.append(p)

        # Set opt
        # Start lr at 0 if warmup #moved to WarmUpScheduler
        #_lr = self.learning_rate
        #print('*************************************************************_lr',_lr)
        _lr= 0 if self.warmup_epochs!=0 else self.learning_rate
        #optimizer = self.optimizer_fn([{'params': net_pars,
        #                                'lr': _lr},
        #                               {'params': pool_pars,
        #                                'weight_decay': 0,
        #                                'lr': _lr}],
        #                              lr=_lr,
        #                              **self.optimizer_kwargs)
        optimizer = self.optimizer_fn([{'params': net_pars},
                                       {'params': pool_pars,'weight_decay': 0}],
                                      lr=_lr,
                                      **self.optimizer_kwargs)

        if self.scheduler_fn is None:
            return optimizer

        logging.info(f'Configuring lr scheduler: {self.scheduler_fn} with {self.scheduler_kwargs}')
        if isinstance(self.scheduler_fn, list):
            schedulers = [sch_fn(optimizer, **sch_kwargs) for sch_fn, sch_kwargs in zip(self.scheduler_fn, self.scheduler_kwargs)]
        else:
            schedulers = [
                {'scheduler': self.scheduler_fn(optimizer, **self.scheduler_kwargs),
                 'frequency': 1,
                 'name': 'main_lr_scheduler'
                 }]

        #if self.warmup_epochs > 0:
        #    schedulers.append({'scheduler': WarmupScheduler(optimizer,
        #                                                    warmup_steps=self.warmup_epochs),
        #                       'frequency': 1,
        #                       'name': 'warmup_scaled_lr'})

        return [optimizer], schedulers

    def configure_callbacks(self):
        if self.param_scheduler_fn is not None:
            logging.info(f'Configuring tau scheduler: {self.param_scheduler_fn} with {self.param_scheduler_kwargs}')
            return [self.param_scheduler_fn('gumbel_tau', **self.param_scheduler_kwargs)]
        return super().configure_callbacks()


from __future__ import unicode_literals
from .session import session
from . import utils
from . import enums
from pyquery import PyQuery as pq
import logging
logger = logging.getLogger(__name__)

@staticmethod
def setup_profile():
    """Initiates a new Steam Profile
    """
    resp = session.get(utils.url_community(
        'profiles', str(utils.get_steam_id())) + 'edit?welcomed=1')
    return resp and resp.ok

@staticmethod
def edit_profile(new_values=None):
    """Updates your Steam profile information.

    Current values are returned if `new_values` is not supplied.

    Supported values for editing are
    --------------------------------
    - `personaName`
    - `real_name`
    - `country`
    - `state`
    - `city`
    - `customURL`
    - `summary`
    - `profile_background`
    - `primary_group_steamid`

    Parameters
    ----------
    new_values : dict, optional
        Dict with editable values, for example::

            {
                'personaName': 'new display name',
                'summary': 'coolest guy ever'
            }

    Returns
    -------
    tuple of (error: str or None, dict)
        `(error, current values)` if `new_values` is not provided,
        else `(error, new values)`.
    """
    def editables(values):
        valid = ['personaName', 'real_name', 'country', 'state', 'city',
                 'customURL', 'summary', 'profile_background', 'primary_group_steamid']

        return {k: v for k, v in list(values.items()) if k in valid}

    def parseForValues(doc):
        values = {}

        all_inputs = doc('#editForm :input').filter(
            lambda i, this: this.tag != "button")
        # visible = all_inputs.filter("[type!='hidden']")
        without_file = all_inputs.filter("[type!='file']")

        for inp in without_file:
            values[inp.name] = inp.value

        return values

    edit_url = utils.url_community(
        'profiles', str(utils.get_steam_id())) + 'edit'
    doc = pq(session.get(edit_url).text)

    values = parseForValues(doc)

    if not new_values:
        return (None, editables(values))
    else:
        values.update(new_values)
        update_resp = pq(session.post(edit_url, data=values).text)
        values = parseForValues(update_resp)
        error = update_resp('#errorText .formRowFields')
        if error:
            return (error.text.strip(), editables(values))

        return (None, editables(values))

@staticmethod
def edit_privacy_settings(new_values=None):
    """Updates your Steam privacy settings.

    Current values are returned if `new_values` is not supplied.

    Supported values for editing are
    --------------------------------
    - `privacySetting`
    - `commentSetting`
    - `inventoryPrivacySetting`
    - `inventoryGiftPrivacy`

    Parameters
    ----------
    new_values : dict, optional
        Dict with editable values, for example::

            {
                'personaName': 'new display name',
                'summary': 'coolest guy ever'
            }

    Returns
    -------
    tuple of (error: str or None, dict)
        `(error, current values)` if `new_values` is not provided,
        else `(error, new values)`.
    """
    def editables(values):
        valid = ['privacySetting', 'commentSetting',
                 'inventoryPrivacySetting', 'inventoryGiftPrivacy']

        return {k: v for k, v in list(values.items()) if k in valid}

    def parseForValues(doc):
        all_inputs = doc('#editForm :input').filter(
            lambda i, this: this.tag != "button")

        values = {inp.name: inp.value for inp in all_inputs}

        for inp in doc('#editForm input:checked'):
            values[inp.name] = inp.value
            if inp.name in ['privacySetting', 'inventoryPrivacySetting']:
                values[inp.name] = enums.PrivacyState(int(inp.value))
            if inp.name == 'commentSetting':
                values[inp.name] = enums.CommentPrivacyState(inp.value)

        if values['inventoryGiftPrivacy'] is not None:
            values['inventoryGiftPrivacy'] = bool(
                int(values['inventoryGiftPrivacy']))
        else:
            values['inventoryGiftPrivacy'] = False

        return values

    edit_url = utils.url_community('profiles', str(
        utils.get_steam_id())) + 'edit/settings'
    doc = pq(session.get(edit_url).text)
    values = parseForValues(doc)

    if not new_values:
        return (None, editables(values))
    else:
        values.update(new_values)
        for k, v in list(editables(values).items()):
            if k == 'inventoryGiftPrivacy':
                values[k] = int(v)
            else:
                values[k] = v.value

        resp = session.post(edit_url, data=values)
        update_resp = pq(resp.text)
        values = parseForValues(update_resp)
        error = update_resp('#errorText .formRowFields')
        if error:
            return (error.text.strip(), editables(values))

        return (None, editables(values))

@staticmethod
def upload_avatar(image):
    """Sets the current account's avatar on Steam.

    Parameters
    ----------
    image : file
        File-like object, in binary mode.
    """
    data = {
        'MAX_FILE_SIZE': 1048576,
        'type': 'player_avatar_image',
        'sId': str(utils.get_steam_id()),
        'sessionid': utils.get_session_id(),
        'doSub': 1,
        'json': 1
    }

    files = {'avatar': image}
    resp = session.post(utils.url_community(
        'actions', 'FileUploader'), files=files, data=data)

    if not resp.ok:
        logger.error('Avatar upload: HTTP error %s', resp.status_code)
        return

    try:
        body = resp.json()
    except:
        body = {}

    if not body or not body.get('success'):
        logger.error('Avatar upload: Malformed response')

    if not body.get('success') and body.get('message'):
        logger.error('Avatar upload: %s', body['message'])

    return body


from time import perf_counter_ns
import cv2
from cv2 import THRESH_BINARY
import numpy as np
import imutils
import pytesseract
import Sudoku
import argparse

input_board = [[0,0,0,0,0,0,0,0,0],
               [0,0,0,0,0,0,0,0,0],
               [0,0,0,0,0,0,0,0,0],
               [0,0,0,0,0,0,0,0,0],
               [0,0,0,0,0,0,0,0,0],
               [0,0,0,0,0,0,0,0,0],
               [0,0,0,0,0,0,0,0,0],
               [0,0,0,0,0,0,0,0,0],
               [0,0,0,0,0,0,0,0,0]]

# Check for arguments in command line
parser = argparse.ArgumentParser()
parser.add_argument('-d', action='store_true')
args = parser.parse_args()

# First match the image with the sudoku board template
# to locate the sudoku board within the template ###########################################################################################
# Read the template
template = cv2.imread('template.jpg')
# Convert the template to grayscale
template = cv2.cvtColor(template, cv2.COLOR_BGR2GRAY)
# Get the Canny edges for increased accuracy
#template = cv2.Canny(template, 50, 200)

# Store width and height of template
(tH, tW) = template.shape[:2]

# Display the template's canny edges
if args.d:
    cv2.imshow("Template", template)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

# Create a mask for the contents of each box
ret, mask = cv2.threshold(template, 200, 255, cv2.THRESH_BINARY)

# Invert the colours of the mask
mask = cv2.bitwise_not(mask)

# Display the Mask
if args.d:
    cv2.imshow("Mask", mask)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

# Now begin multiscaling the image
# Read the main image
img_rgb = cv2.imread('pic.jpg')

# Convert it to grayscale
img_gray = cv2.cvtColor(img_rgb, cv2.COLOR_BGR2GRAY)

# Keep track of the region and scale of the image with the best match
found = None

# Loop over the scales of the image
for scale in np.linspace(0.2, 1.0, 40)[::-1] :
    # resize the image according to the scale, and keep track
    # of the ratio of the resizing
    resized = imutils.resize(img_gray, width = int(img_gray.shape[1] * scale))
    r = img_gray.shape[1] / float(resized.shape[1])

    # if the resized image is smaller than the template, then break
    # from the loop
    if resized.shape[0] < tH or resized.shape[1] < tW:
        break

    # detect edges in the resized, grayscale image and apply template
    # matching to find the template in the image
    result = cv2.matchTemplate(resized, template, cv2.TM_SQDIFF, None, mask)
    (minVal, _, minLoc, _) = cv2.minMaxLoc(result)

    # Visualize the iteration
    # draw a bounding box around the detected region
    if args.d:
        print("Smaller value = closer the match")
        print(minVal)
        clone = np.dstack([resized, resized, resized])
        cv2.rectangle(clone, (minLoc[0], minLoc[1]),
            (minLoc[0] + tW, minLoc[1] + tH), (0, 0, 255), 2)
        cv2.imshow("Visualize", clone)
        cv2.waitKey(0)

    # if we have found a new maximum correlation value, then update
    # the bookkeeping variable
    if found is None or minVal < found[0]:
        found = (minVal, minLoc, r)

# unpack the bookkeeping variable and compute the (x, y) coordinates
# of the bounding box based on the resized ratio
(_, minLoc, r) = found
(startX, startY) = (int(minLoc[0] * r), int(minLoc[1] * r))
(endX, endY) = (int((minLoc[0] + tW) * r), int((minLoc[1] + tH) * r))

# draw a bounding box around the detected result and display the image
if args.d:
    cv2.rectangle(img_rgb, (startX, startY), (endX, endY), (0, 0, 255), 2)
    cv2.imshow("Image", img_rgb)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

# Next we will detect the numbers that are found within the bounding box #################################################################
pytesseract.pytesseract.tesseract_cmd = 'C:\\Program Files\\Tesseract-OCR\\tesseract.exe'

# Crop the image to only read the sudoku box
img_crop = img_rgb[startY:endY, startX:endX].copy()

# Resizing to a larger image will help tesseract detect the numbers
img_crop = cv2.resize(img_crop, None, fx=1.3, fy=1.3, interpolation=cv2.INTER_CUBIC)

# Tesseract only works with RGB images, OpenCV uses BGR
img_text = cv2.cvtColor(img_crop, cv2.COLOR_BGR2GRAY)

hImg, wImg = img_text.shape

# Define the size of each box in the sudoku grid
boxW = int(wImg/9)
boxH = int(hImg/9)

# Remove horizontal and vertical lines
kernel_vertical = cv2.getStructuringElement(cv2.MORPH_RECT, (1,50))
temp1 = 255 - cv2.morphologyEx(img_text, cv2.MORPH_CLOSE, kernel_vertical)
horizontal_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (50,1))
temp2 = 255 - cv2.morphologyEx(img_text, cv2.MORPH_CLOSE, horizontal_kernel)
temp3 = cv2.add(temp1, temp2)
result = cv2.add(temp3, img_text)
img_text = result

img_text = cv2.medianBlur(img_text, 5)

img_text = cv2.adaptiveThreshold(img_text,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,\
            cv2.THRESH_BINARY,11,2)

boxes = pytesseract.image_to_boxes(img_text, config="--psm 6 -c tessedit_char_whitelist=0123456789 --tessdata-dir 'C:\\Program Files\\Tesseract-OCR\\tessdata'")

for b in boxes.splitlines() :
    b = b.split(' ')
    print(b)
    x, y, w, h = int(b[1]), int(b[2]), int(b[3]), int(b[4])
    cv2.rectangle(img_text, (x,hImg-y), (w,hImg-h), (0,0,255), 1)
    cv2.putText(img_text, b[0], (x, hImg-y+25), cv2.FONT_HERSHEY_PLAIN, 1, (50, 50, 255), 2)

    # Get the coordinates of the found digit
    centerx = int((x+w)/2)
    centery = int(hImg - (y+h)/2)

    # Check which row the digit is in
    row = 0
    for row_ in range(9) :
        if centerx < (row_ * boxW + boxW) :
            row = row_
            break

    # Check which col the digit is in
    col = 0
    for col_ in range(9) :
        if centery < (col_ * boxH + boxH) :
            col = col_
            break

    # Enter the digit into our board
    input_board[col][row] = int(b[0])

# Display the detected board in the terminal
print("\nThe input board is:")
print(np.matrix(input_board))

# Show the detected digits
if args.d:
    cv2.imshow("Text", img_text)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

# Solve the sudoku puzzle
print("\nThe solved board is:")
Sudoku.solve(input_board)

# Remove unnecessary characters from text document
with open(r'Solution.txt', 'r') as infile, \
     open(r'Formatted_Solution.txt', 'w') as outfile:
    data = infile.read()
    data = data.replace("[", " ")
    data = data.replace("]", " ")
    outfile.write(data)

# Generate an array from the text document
solved_board = np.genfromtxt("Formatted_Solution.txt", dtype=int)

# Iterate through the rows and columns, entering the solutions into the empty spaces
for row in range (9) :
    for col in range (9) :
        Xcoord = int(row * boxW + 0.25 * boxW)
        Ycoord = int(col * boxH + 0.75 * boxH)

        if input_board[col][row] == 0:
            cv2.putText(img_crop, str(solved_board[col][row]), (Xcoord, Ycoord), cv2.FONT_HERSHEY_DUPLEX, 3, (50, 50, 255), 2)

cv2.imshow("Solution", img_crop)

cv2.waitKey(0)
cv2.destroyAllWindows()


import click
import numpy as np
from copy import deepcopy
from matplotlib import pyplot as plt

def cost_function(x_true, x_em):
    """
    :param x_true: The pixels of original image (normalized). Each row is an RGB vector.
    :param x_em: The image that comes from the training of em.
    :return: The error between the real and the segmented image.
    """

    n = x_true.shape[0]  # The The number of our examples (pixels)

    return 1/n * np.sum(np.linalg.norm(x_true - x_em)**2)

def construct_image(height, width, g, m):
    """
    :param height: The height of the image
    :param width:  The width of the image
    :param g: Table (N x K). Contains the posterior probabilities of each example n belongs to each one
    of the K categories-segments.
    :param m: Table (K x 3). Contains an average vector (RGB color) from the data that belongs
    on the k-th category-segment
    :return: The normalized image which is needed to compute the error of the expectation maximisation algorithm and
    the colored image which is produced from the algorithm.
    """
    new_image = np.zeros((height * width, 3))

    for n in range(g.shape[0]):
        k = g[n].argmax()
        new_image[n] = m[k]

    flat = deepcopy(new_image)

    new_image = new_image.reshape((height, width, 3))

    k = m.shape[0]

    plt.imshow(new_image)
    plt.savefig('em_{}'.format(k))
    plt.show()

    return flat, new_image

def gaussian_mixture(x, p, m, s, k):
    """
    :param x: Our data (pixels). Each row is an RGB vector of a pixel
    :param p: Table (N x K). Contains the prior probabilities of each example n belongs to each category-segment k
    :param m: Table (K x 3). Contains an average vector(RGB) of the color from the data that belongs
    on category-segment k
    :param s: The covariance table S.
    :param k: The number of the categories.
    :return:  Table of shape (N x K) that contains the probabilities (that comes for mixture of Gaussian distributions)
    for each example n (pixel) belongs to each one of the K categories
    """
    probabilities = np.zeros((x.shape[0], k))

    for k_i in range(k):

        first_part = 1 / np.sqrt(2 * np.pi * s[k_i])
        second_part = np.exp(-(1 / (2 * s[k_i])) * (x - m[k_i, :]) ** 2)

        probabilities[:, k_i] = p[k_i] * np.prod(first_part * second_part, axis=1)

    return np.array(probabilities)

def log_likelihood(probabilities):
    return np.sum(np.log(np.sum(probabilities, axis=1)))

def maximization_step(x, g):
    """
    Execute the maximization stem of the algorithm and update the parameters
    :param x: Our data (pixels). Each row is an RGB vector of a pixel
    :param g: Table (N x K). Contains the posterior probabilities of each example n belongs to each one of
    the K categories-segments
    :return: The updated parameters.
    p: Table (N x K). Contains the prior probabilities of each example n belongs to each category-segment k
    m: Table (K x 3). Contains an average vector(RGB) of the color from the data that belongs on category-segment k
    s: The covariance table S.
    """

    k = g.shape[1]

    m = np.zeros((k, x.shape[1]))
    p = np.zeros(k)
    s = np.zeros(k)

    for k_i in range(k):
        g_k = g[:, k_i].reshape((-1, 1))

        m[k_i, :] = np.sum(g_k * x, axis=0) / np.sum(g_k)

        s[k_i] = np.sum(np.sum(g_k * ((x - m[k_i]) ** 2), axis=1)) / (x.shape[1] * np.sum(g_k))

        p[k_i] = np.sum(g_k) / x.shape[0]

    return p, m, s

def expectation_step(probabilities):
    """
    Execute the expectation step of the algorithm
    :param probabilities: Table (N x K). Contains the probabilities (that comes for Gaussian mixture)
    of each example n belongs to each category-segment k
    :return: Table (N x K). Contains the posterior probabilities of each example n belongs to each one of
    the K categories-segments
    """

    denominator = np.sum(probabilities, axis=1)

    return probabilities / denominator.reshape((-1, 1))

def initialize_parameters(k, d):
    """
    :param k: The number of categories-segments
    :param d: The dimension of each example-pixel (R, G, B) = 3
    :return: The prior probabilities, the average_vectors and the covariance table
    """

    # At the start, the prior probability of each category is the same and equal to 1/k
    prior = np.full(k, 1/k)

    # We have average_vector of d-dimension for each category and initialize
    # them with values from 0-1 because we have a normalized image.
    m = np.zeros((k, d))

    for i in range(m.shape[0]):
        m[i, :] = np.random.uniform(0, 1, d)

    # Initialize the covariance of each category
    # with values between 0.2-0.8 -> 60% of the real values
    s = np.random.uniform(0.2, 0.8, k)

    return prior, m, s

def expectation_maximization(x, k, iterations, tolerance):
    """
    :param x: Table dimension (N x 3) with the pixels of th image
    :param k: The number o categories-segments
    :param tolerance: The tolerance you accept
    :param iterations: The number of iterations you want to run the algorithm
    :return: The probabilities of N example belongings on the k category and
    the average vectors of each category
    """
    d = x.shape[1]  # The dimension of each example-pixel (R G B) = 3

    prior, m, s = initialize_parameters(k, d)

    prob = gaussian_mixture(x, prior, m, s, k)

    for t in range(iterations):

        log_likelihood_old = log_likelihood(prob)

        g = expectation_step(prob)

        prior, m, s = maximization_step(x, g)

        prob = gaussian_mixture(x, prior, m, s, k)

        log_likelihood_new = log_likelihood(prob)
        print('log_likelihood of {:<3} iteration: {}'.format(t, log_likelihood_new))

        if log_likelihood_new - log_likelihood_old < 0:
            print('Error in coding')

        if np.abs(log_likelihood_new - log_likelihood_old) < tolerance:
            print('Converged')
            break

    return g, m

@click.command()
@click.option('--segments', default=8)
@click.option('--path', default='im.jpg')
@click.option('--iterations', default=100)
@click.option('--tolerance', default=1e-6)
def main(segments, path, iterations, tolerance):
    img = plt.imread(path)

    print('Image shape: {}'.format(img.shape))

    #  Calculate our N independent pixels
    number_of_pixels = img.shape[0] * img.shape[1]

    #  Reshape the image in order to have a table (N x 3)
    data = img.reshape((number_of_pixels, 3))

    # Normalize our data
    data = data / 255

    post_probabilities, average_vectors = expectation_maximization(x=data,
                                                                   k=segments,
                                                                   iterations=iterations,
                                                                   tolerance=tolerance)

    flt, new_img = construct_image(img.shape[0], img.shape[1], post_probabilities, average_vectors)

    error = cost_function(data, flt)

    print('Total error: {}'.format(error))

if __name__ == '__main__':
    main()


# Zachary Hoover || Create Formated File v1.0
# 10-17-22

import shutil
from datetime import datetime

template = r'baseTemplate.py'

print("\n    File Creator")
print(" --------+---------   \n")
fileName = input(" Enter the name of the file: ")
AssignmentName = input(" Enter the assignment title: ")
path = input(" Enter file path(leave blank for current dir): ")

if path == "":
    path = (fileName + ".py")

shutil.copyfile(template, path)

with open(path, 'r') as file :
  filedata = file.read()

date = datetime.today().strftime('%m-%d-%y')
# Replace the target string
filedata = filedata.replace('{date}', date)
filedata = filedata.replace('{assignmentName}', AssignmentName)
filedata = filedata.replace('# import {moduleName} as foo', f"import {fileName} as foo")
filedata = filedata.replace('# {list = getmembers(foo, isfunction)}', "list = getmembers(foo, isfunction)")

header = r"\n    " + AssignmentName + r"   "
filedata = filedata.replace('{assignmentTitle}', header)
bottom = r" "

i = 0
while i < (len(header)/2):
	bottom += r"-"
	i += 1
bottom += r"+"
i = 0
while i < (len(header)/2):
	bottom += r"-"
	i += 1

filedata = filedata.replace('{dashThing}', bottom)

# Write the file out again
with open(path, 'w') as file:
  file.write(filedata)