import numpy as np
from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.feature_extraction.text import CountVectorizer
from tqdm import tqdm
from typing import List, Union
import warnings
from .mmr import mmr
from .maxsum import max_sum_similarity

class KeyBERT:
    """
    A minimal method for keyword extraction with BERT

    The keyword extraction is done by finding the sub-phrases in
    a document that are the most similar to the document itself.

    First, document embeddings are extracted with BERT to get a
    document-level representation. Then, word embeddings are extracted
    for N-gram words/phrases. Finally, we use cosine similarity to find the
    words/phrases that are the most similar to the document.

    The most similar words could then be identified as the words that
    best describe the entire document.

    Arguments:
        model: the name of the model used by sentence-transformer
               for a full overview see https://www.sbert.net/docs/pretrained_models.html

    """
    def __init__(self, model: str = 'distilbert-base-nli-mean-tokens'):
        self.model = SentenceTransformer(model)
        self.doc_embeddings = None

    def extract_query_numpy_embedding(self, query: str):
        return self.model.encode(query, convert_to_tensor=False)

    def extract_query_tensor_embedding(self, query: str):
        return self.model.encode(query, convert_to_tensor=True)

    def extract_keywords(self,
                         docs: Union[str, List[str]],
                         keyphrase_length: int = 1,
                         stop_words: Union[str, List[str]] = 'english',
                         top_n: int = 5,
                         min_df: int = 1,
                         use_maxsum: bool = False,
                         use_mmr: bool = False,
                         diversity: float = 0.5,
                         nr_candidates: int = 20) -> Union[List[str], List[List[str]]]:
        """ Extract keywords/keyphrases

        NOTE:
            I would advise you to iterate over single documents as they
            will need the least amount of memory. Even though this is slower,
            you are not likely to run into memory errors.

        Multiple Documents:
            There is an option to extract keywords for multiple documents
            that is faster than extraction for multiple single documents.

            However...this method assumes that you can keep the word embeddings
            for all words in the vocabulary in memory which might be troublesome.

            I would advise against using this option and simply iterating
            over documents instead if you have limited hardware.

        Arguments:
            docs: The document(s) for which to extract keywords/keyphrases
            keyphrase_length: Length, in words, of the extracted keywords/keyphrases
            stop_words: Stopwords to remove from the document
            top_n: Return the top n keywords/keyphrases
            min_df: Minimum document frequency of a word across all documents
                    if keywords for multiple documents need to be extracted
            use_maxsum: Whether to use Max Sum Similarity for the selection
                        of keywords/keyphrases
            use_mmr: Whether to use Maximal Marginal Relevance (MMR) for the
                     selection of keywords/keyphrases
            diversity: The diversity of the results between 0 and 1 if use_mmr
                       is set to True
            nr_candidates: The number of candidates to consider if use_maxsum is
                           set to True

        Returns:
            keywords: the top n keywords for a document

        """

        if isinstance(docs, str):
            return self._extract_keywords_single_doc(docs,
                                                     keyphrase_length,
                                                     stop_words,
                                                     top_n,
                                                     use_maxsum,
                                                     use_mmr,
                                                     diversity,
                                                     nr_candidates)
        elif isinstance(docs, list):
            warnings.warn("Although extracting keywords for multiple documents is faster "
                          "than iterating over single documents, it requires significant memory "
                          "to hold all word embeddings. Use this at your own discretion!")
            return self._extract_keywords_multiple_docs(docs,
                                                        keyphrase_length,
                                                        stop_words,
                                                        top_n,
                                                        min_df=min_df)

    def _extract_keywords_single_doc(self,
                                     doc: str,
                                     keyphrase_length: int = 1,
                                     stop_words: Union[str, List[str]] = 'english',
                                     top_n: int = 5,
                                     use_maxsum: bool = False,
                                     use_mmr: bool = False,
                                     diversity: float = 0.5,
                                     nr_candidates: int = 20) -> List[str]:
        """ Extract keywords/keyphrases for a single document

        Arguments:
            doc: The document for which to extract keywords/keyphrases
            keyphrase_length: Length, in words, of the extracted keywords/keyphrases
            stop_words: Stopwords to remove from the document
            top_n: Return the top n keywords/keyphrases
            use_mmr: Whether to use Max Sum Similarity
            use_mmr: Whether to use MMR
            diversity: The diversity of results between 0 and 1 if use_mmr is True
            nr_candidates: The number of candidates to consider if use_maxsum is set to True

        Returns:
            keywords: The top n keywords for a document

        """
        try:
            # Extract Words
            n_gram_range = (keyphrase_length, keyphrase_length)
            count = CountVectorizer(ngram_range=n_gram_range, stop_words=stop_words).fit([doc])
            words = count.get_feature_names()

            # Extract Embeddings
            doc_embedding = self.model.encode([doc])
            word_embeddings = self.model.encode(words)

            # Calculate distances and extract keywords
            if use_mmr:
                keywords = mmr(doc_embedding, word_embeddings, words, top_n, diversity)
            elif use_maxsum:
                keywords = max_sum_similarity(doc_embedding, word_embeddings, words, top_n, nr_candidates)
            else:
                distances = cosine_similarity(doc_embedding, word_embeddings)
                keywords = [words[index] for index in distances.argsort()[0][-top_n:]][::-1]

            return keywords
        except ValueError:
            return []

    def _extract_keywords_multiple_docs(self,
                                        docs: List[str],
                                        keyphrase_length: int = 1,
                                        stop_words: str = 'english',
                                        top_n: int = 5,
                                        min_df: int = 1):
        """ Extract keywords/keyphrases for a multiple documents

        This currently does not use MMR as

        Arguments:
            docs: The document for which to extract keywords/keyphrases
            keyphrase_length: Length, in words, of the extracted keywords/keyphrases
            stop_words: Stopwords to remove from the document
            top_n: Return the top n keywords/keyphrases
            min_df: The minimum frequency of words

        Returns:
            keywords: The top n keywords for a document

        """
        # Extract words
        n_gram_range = (keyphrase_length, keyphrase_length)
        count = CountVectorizer(ngram_range=n_gram_range, stop_words=stop_words, min_df=min_df).fit(docs)
        words = count.get_feature_names()
        df = count.transform(docs)

        # Extract embeddings
        word_embeddings = self.model.encode(words, show_progress_bar=True)
        doc_embeddings = self.model.encode(docs, show_progress_bar=True)

        # Extract keywords
        keywords = []
        for index, doc in tqdm(enumerate(docs)):
            doc_words = [words[i] for i in df[index].nonzero()[1]]

            if doc_words:
                doc_word_embeddings = np.array([word_embeddings[i] for i in df[index].nonzero()[1]])
                distances = cosine_similarity([doc_embeddings[index]], doc_word_embeddings)[0]
                doc_keywords = [doc_words[i] for i in distances.argsort()[-top_n:]]
                keywords.append(doc_keywords)
            else:
                keywords.append(["None Found"])

        return keywords


from django.forms import ModelForm
from selfie.models import Selfie

class SelfieForm(ModelForm):
    class Meta:
        model = Selfie
        fields = ['selfie_before', 'selfie_after']

    def __init__(self, *args, **kwargs):
        self.request = kwargs['initial']['request']

        super(SelfieForm, self).__init__(*args, **kwargs)
        for myField in self.fields:
            self.fields[myField].widget.attrs['class'] = 'form-control'

    def save(self, *args, **kwargs):
        kwargs['commit']=False

        obj = super(SelfieForm, self).save(*args, **kwargs)

        if self.request:
            obj.user_id = self.request.user.id

        obj.save()

        return obj


import pytest
import time
# import logging
from typing import List
from threading import Thread

from hydrand.config import N
from hydrand.data import Phase
from hydrand.node import Node

from hydrand.test_utils import AnyNode, LeaderNode, NoLeaderNode, force_close_port

import hydrand.node as node_module

STARTUP_DELAY = 5.0

node: AnyNode  # any call to this node is multiplexed to all other nodes
leader: LeaderNode
noleader: NoLeaderNode
nodes: List[Node]

@pytest.fixture(autouse=True)
def nodes_setup_and_teardown():
    global node, nodes, leader, noleader
    force_close_port(5000)

    node_module.PROTOCOL_START_TIME = time.time() + STARTUP_DELAY
    nodes = [Node(i) for i in range(N)]

    node = AnyNode(nodes)
    leader = LeaderNode(nodes)
    noleader = NoLeaderNode(nodes)

    node.start_listening()
    time.sleep(STARTUP_DELAY / 2)
    node.connect()
    time.sleep(STARTUP_DELAY / 2)

    yield
    print("TEST TEARDOWN")
    node.shutdown()

def test_start_nodes():
    assert node.round == 0
    assert node.phase == Phase.Propose
    assert node.connected

def test_initial_round_leader():
    node.advance_round()
    node.compute_leader()
    assert node.leaders.all_equal

def test_propose_basics():
    node.advance_round()
    node.compute_leader()
    leader.propose()

    assert leader.phase == Phase.Vote
    assert noleader.phase == Phase.Propose

    assert noleader.next_message is None
    noleader.receive_message()
    assert all(noleader.next_message.values)

    assert noleader.process_message()
    assert noleader.phase == Phase.Acknowledge

    assert node.beacon.all_equal

def test_reveal_verification():
    node.advance_round()
    node.compute_leader()
    leader.propose()
    signed_msg = leader.sent_messages[0]
    msg = signed_msg.message
    dataset = msg.dataset
    assert leader.verify_revealed_secret(dataset.revealed_secret)
    assert noleader.verify_revealed_secret(dataset.revealed_secret)

def test_round_step():

    node.advance_round()
    node.compute_leader()

    leader.propose()
    noleader.receive_messages()
    noleader.process_message()  # acks are sent here

    assert noleader.phase == Phase.Acknowledge
    node.receive_messages()
    for n in noleader:
        assert len(n._message_queue) == N - 2
    assert len(leader._message_queue) == N - 1
    node.process_messages()  # confirms are sent here

    node.receive_messages()
    node.process_messages()

def test_2rounds_step():
    node.advance_round()
    node.compute_leader()

    leader.propose()

    node.receive_messages()
    node.process_messages()  # sends out acks

    node.receive_messages()
    node.process_messages()  # sends out confirms

    node.receive_messages()
    node.process_messages()

    # all nodes have beacon and confirmation certificate here

    node.advance_round()
    node.compute_leader()

    assert node.leader.all_equal
    assert node.round == 2

    leader.propose()
    node.receive_messages()
    node.process_messages()

def test_run_round():
    threads = [Thread(target=n.run_round) for n in nodes]
    for t in threads:
        t.start()

    for t in threads:
        t.join()

    assert node.beacon.all_equal
    assert node.beacon is not None

    for cc in node.confirmation_certificate:
        assert cc is not None

def test_recover_first_round():
    node.advance_round()
    node.compute_leader()

    leader.compute_beacon(leader.shared_secret)

    node.update_phase()
    noleader.recover()

    noleader.receive_messages()
    noleader.process_messages()

    assert node.beacon.all_equal


import gtk
import gobject
import os, sys
import thread
from os.path import exists, join
from time import time

from pynicotine.utils import _
from dirchooser import ChooseDir
from utils import OpenUri, InitialiseColumns, recode, HumanSize
dir_location = os.path.dirname(os.path.realpath(__file__))

def dirstats(directory):
	from os.path import join, getsize
	totaldirs = 0
	totalfiles = 0
	totalsize = 0
	extensions = {}
	for root, dirs, files in os.walk(directory):
		totaldirs += len(dirs)
		totalfiles += len(files)
		for f in files:
			try:
				totalsize += getsize(join(root, f))
			except OSError:
				pass
			parts = f.rsplit('.', 1)
			if len(parts) == 2 and len(parts[1]) < 5:
				try:
					extensions[parts[1]] += 1
				except KeyError:
					extensions[parts[1]] = 1
	return totaldirs, totalfiles, totalsize, extensions
def widgetname(widget):
	"""Raises AttributeError if given widget has no name property whatsoever."""
	try:
		return gtk.Buildable.get_name(widget) # New GTK way
	except TypeError:
		return widget.get_name() # Old GTK way

class FastConfigureAssistant(object):
	def __init__(self, frame):
		self.frame = frame
		self.initphase = True # don't respond to signals unless False
		self.config = frame.np.config
		builder = gtk.Builder()
		builder.add_from_file(join(dir_location, "fastconfigure.glade"))
		self.window = builder.get_object("FastConfigureAssistant")
		builder.connect_signals(self)
		self.kids = {}
		for i in builder.get_objects():
			try:
				self.kids[widgetname(i)] = i
			except AttributeError:
				pass
		numpages = self.window.get_n_pages()
		for n in xrange(numpages):
			page = self.window.get_nth_page(n)
			template = self.window.get_page_title(page)
			self.window.set_page_title(page, template % {'page':(n+1), 'pages':numpages})
		self.templates = {
				'listenport': self.kids['listenport'].get_text(),
			}
		# Page specific, sharepage
		# the lsat column is the raw byte/unicode object for the folder (not shown)
		self.sharelist = gtk.ListStore(gobject.TYPE_STRING, gobject.TYPE_STRING, gobject.TYPE_STRING, gobject.TYPE_STRING, gobject.TYPE_STRING, gobject.TYPE_STRING)
		columns = InitialiseColumns(self.kids['shareddirectoriestree'],
			[_("Directory"), 0, "text"],
			[_("Size"), 0, "text"],
			[_("Files"), 0, "text"],
			[_("Dirs"), 0, "text"],
			[_("File types"), 0, "text"],
		)
		self.kids['shareddirectoriestree'].set_model(self.sharelist)
		self.kids['shareddirectoriestree'].get_selection().set_mode(gtk.SELECTION_MULTIPLE)
		self.initphase = False
		#self.show() # DEBUG
		#self.window.set_current_page(3) # DEBUG
	def show(self):
		self.initphase = True
		self._populate()
		self.initphase = False
		self.window.show()
	def _populate(self):
		# userpasspage
		self.kids['username'].set_text(self.config.sections["server"]["login"])
		self.kids['password'].set_text(self.config.sections["server"]["passw"])
		# portpage
		self.kids['advancedports'].set_expanded(self.config.sections["server"]["upnp"])
		if (time() - self.config.sections["server"]["lastportstatuscheck"]) > (60 * 60 * 24 * 31) or self.config.sections["server"]["upnp"]:
			# More than a month ago since our last port status check
			self.kids['portopen'].set_active(False)
			self.kids['portclosed'].set_active(False)
			self.kids['porthidden'].set_active(True)
		else:
			if self.config.sections["server"]["firewalled"]:
				self.kids['portclosed'].set_active(True)
			else:
				self.kids['portopen'].set_active(True)
		self.kids['lowerport'].set_value(self.config.sections["server"]["portrange"][0])
		self.kids['upperport'].set_value(self.config.sections["server"]["portrange"][1])
		self.kids['useupnp'].set_active(self.config.sections["server"]["upnp"])
		# sharepage
		if self.config.sections['transfers']['downloaddir']:
			self.kids['downloaddir'].set_current_folder(self.config.sections['transfers']['downloaddir'])
		self.sharelist.clear()
		if self.config.sections["transfers"]["friendsonly"] and self.config.sections["transfers"]["enablebuddyshares"]:
			for directory in self.config.sections["transfers"]["buddyshared"]:
				self.addshareddir(directory)
		else:
			for directory in self.config.sections["transfers"]["shared"]:
				self.addshareddir(directory)
		self.kids['onlysharewithfriends'].set_active(self.config.sections["transfers"]["friendsonly"])
		# If the user has a public share and a buddy share we cannot update this from FastConfigure (we only have 1 list of dirs)
		self.caneditshare = (not self.config.sections["transfers"]["enablebuddyshares"] or self.config.sections["transfers"]["friendsonly"])
		self.kids['onlysharewithfriends'].set_sensitive(self.caneditshare)
		self.kids['addshare'].set_sensitive(self.caneditshare)
		self.kids['removeshares'].set_sensitive(self.caneditshare)
		self.kids['shareddirectoriestree'].set_sensitive(self.caneditshare)
	def store(self):
		# userpasspage
		self.config.sections["server"]["login"] = self.kids['username'].get_text()
		self.config.sections["server"]["passw"] = self.kids['password'].get_text()
		# portpage
		self.config.sections['server']['firewalled'] = not self.kids['portopen'].get_active()
		self.config.sections['server']['lastportstatuscheck'] = time()
		# sharepage
		self.config.sections['transfers']['downloaddir'] = self.kids['downloaddir'].get_current_folder()
		if self.caneditshare:
			self.config.sections["transfers"]["friendsonly"] = self.kids['onlysharewithfriends'].get_active()
			if self.config.sections["transfers"]["friendsonly"] and self.config.sections["transfers"]["enablebuddyshares"]:
				self.config.sections["transfers"]["buddyshared"] = self.getshareddirs()
			else:
				self.config.sections["transfers"]["shared"] = self.getshareddirs()
	def OnClose(self, widget):
		self.window.hide()
	def OnApply(self, widget):
		self.store()
		self.window.hide()
		if not self.frame.np.serverconn:
			self.frame.OnFirstConnect(-1)
	def OnCancel(self, widget):
		self.window.hide()
	def updatepage(self, page):
		"""Updates information on the given page with. Use _populate if possible"""
		if not page:
			return
		name = widgetname(page)
		if name == 'portpage':
			self.kids['listenport'].set_markup(_(self.templates['listenport']) % {'listenport':'<b>'+str(self.frame.np.waitport)+'</b>'})
		#else:
		#	print "Unknown page %s" % (name,)
	def resetcompleteness(self, page = None):
		"""Turns on the complete flag if everything required is filled in."""
		# Never use self.config.sections here, only self.kids.
		complete = False
		if not page:
			pageid = self.window.get_current_page()
			page = self.window.get_nth_page(pageid)
			if not page:
				return
		name = widgetname(page)
		if name == 'welcomepage':
			complete = True
		elif name == 'userpasspage':
			if (len(self.kids['username'].get_text()) > 0 and
				len(self.kids['password'].get_text()) > 0):
				complete = True
		elif name == 'portpage':
			if self.kids['useupnp'].get_active():
				complete = True
			else:
				if self.kids['portopen'].get_active() or self.kids['portclosed'].get_active():
					complete = True
		elif name == 'sharepage':
			if exists(self.kids['downloaddir'].get_filename()):
				complete = True
		elif name == 'summarypage':
			complete = True
			showcpwarning = (self.kids['portclosed'].get_active() and not self.kids['useupnp'].get_active())
			if showcpwarning:
				self.kids['labelclosedport'].show()
				self.kids['warningclosedport'].show()
			else:
				self.kids['labelclosedport'].hide()
				self.kids['warningclosedport'].hide()
			shownfwarning = (self.kids['onlysharewithfriends'].get_active() or len(self.getshareddirs()) == 0)
			if shownfwarning:
				self.kids['labelnoshare'].show()
				self.kids['warningnoshare'].show()
			else:
				self.kids['labelnoshare'].hide()
				self.kids['warningnoshare'].hide()
		#else:
		#	print "Unknown page with name " + name
		self.window.set_page_complete(page, complete)
	def OnPrepare(self, widget, page):
		self.window.set_page_complete(page, False)
		self.updatepage(page)
		self.resetcompleteness(page)
	def OnEntryChanged(self, widget, param1 = None, param2 = None, param3 = None):
		name = widgetname(widget)
		#print "Changed %s, %s" % (widget, name)
		self.resetcompleteness()
	def getshareddirs(self):
		iter = self.sharelist.get_iter_root()
		dirs = []
		while iter is not None:
			dirs.append(self.sharelist.get_value(iter, 5))
			iter = self.sharelist.iter_next(iter)
		return dirs
	def addshareddir(self, directory):
		iter = self.sharelist.get_iter_root()
		while iter is not None:
			if directory == self.sharelist.get_value(iter, 5):
				return
			iter = self.sharelist.iter_next(iter)
		self.sharelist.append([recode(directory), "", "", "", _("Counting files..."), directory])
		thread.start_new_thread(self._addsharedir, (directory,))
	def _addsharedir(self, directory):
		subdirs, files, size, extensions = dirstats(directory)
		exts = []
		for ext, count in extensions.iteritems():
			exts.append((count, ext))
		exts.sort(reverse=True)
		extstring = ", ".join(["%s %s" % (count, ext) for count, ext in exts[:5]])
		if len(exts) > 5:
			extstring += ", ..."
		gobject.idle_add(self._updatedirstats, directory, recode(directory), HumanSize(size), files, subdirs, extstring)
	def _updatedirstats(self, directory, directorystring, size, files, subdirs, extensions):
		iter = self.sharelist.get_iter_root()
		while iter is not None:
			if directory == self.sharelist.get_value(iter, 5):
				self.sharelist.insert_after(iter, [recode(directory), HumanSize(size), files, subdirs, extensions, directory])
				self.sharelist.remove(iter)
				return
			iter = self.sharelist.iter_next(iter)
	def OnButtonPressed(self, widget):
		if self.initphase:
			return
		name = widgetname(widget)
		#print "Pressed %s" % (name)
		if name == "checkmyport":
			OpenUri('='.join(['http://tools.slsknet.org/porttest.php?port', str(self.frame.np.waitport)]))
		if name == "addshare":
			selected = ChooseDir(self.window.get_toplevel(), title=_("Nicotine+")+": "+_("Add a shared directory"))
			if selected:
				for directory in selected:
					self.addshareddir(directory)
		if name == "removeshares":
			model, paths = self.kids['shareddirectoriestree'].get_selection().get_selected_rows()
			refs = [gtk.TreeRowReference(model, x) for x in paths]
			for i in refs:
				self.sharelist.remove(self.sharelist.get_iter(i.get_path()))
		self.resetcompleteness()
	def OnToggled(self, widget):
		name = widgetname(widget)
		if name == 'useupnp':
			# Setting active state
			if widget.get_active():
				self.kids['portopen'].set_inconsistent(True)
				self.kids['portclosed'].set_inconsistent(True)
			else:
				self.kids['portopen'].set_inconsistent(False)
				self.kids['portclosed'].set_inconsistent(False)
			# Setting sensitive state
			inverse = not widget.get_active()
			self.kids['portopen'].set_sensitive(inverse)
			self.kids['portclosed'].set_sensitive(inverse)
			self.kids['checkmyport'].set_sensitive(inverse)
			self.resetcompleteness()
		if self.initphase:
			return
		# Setting changing
		#print "toggled on " + name
		self.resetcompleteness()
	def OnSpinbuttonChangeValue(self, widget, scrolltype):
		if self.initphase:
			return
		name = widgetname(widget)
		#print "pre spinval on " + name
		self.resetcompleteness()
	def OnSpinbuttonValueChanged(self, widget):
		if self.initphase:
			return
		name = widgetname(widget)
		#print "post spinval on " + name
		if name == "lowerport":
			if widget.get_value() > self.kids['upperport'].get_value():
				self.kids['upperport'].set_value(widget.get_value())
		if name == "upperport":
			if widget.get_value() < self.kids['lowerport'].get_value():
				self.kids['lowerport'].set_value(widget.get_value())
		self.resetcompleteness()
	def OnKeyPress(self, widget, event):
		if event.keyval == gtk.gdk.keyval_from_name("Esc"):
			self.OnCancel(None)


# -*- coding: utf-8 -*-
# Load OpenAI Gym and other necessary packages
import gym
import random
import numpy

# Environment
env = gym.make("Taxi-v3")
env.render()
# Training parameters for Q learning
alpha = 0.9 # Learning rate
gamma = 0.9 # Future reward discount factor
num_of_episodes = 2000
num_of_steps = 100 # per each episode

#Parameters for decreasing the exploration rate.
epsilon = 1
max_epsilon = 1
min_epsilon = 0.01
decay_rate = 0.01

tr_rewards = []
tr_actions = []

#Starting with all zeros.
Q_table = numpy.zeros((500, 6))
# Training
for i in range(num_of_episodes):
    state = env.reset()
    tot_reward = 0
    tot_actions = 0
    for j in range(num_of_steps):
        random_val = random.uniform(0, 1)
        if (random_val < epsilon):
            action = random.randint(0, 5)
        else:
            action = numpy.argmax(Q_table[state])

        tot_actions += 1
        next_state, reward, done, info = env.step(action)

        Q_table[state, action] = Q_table[state, action] + alpha * (reward + gamma * numpy.max(Q_table[next_state]) - Q_table[state, action])
        tot_reward += reward
        state = next_state

        if done == True:
            print(tot_reward)
            break

    #Decreasing the exploration rate.
    epsilon = min_epsilon + (max_epsilon - min_epsilon)*numpy.exp(-decay_rate*num_of_episodes)

    tr_actions.append(tot_actions)
    tr_rewards.append(tot_reward)

print(f'Average of total reward: {numpy.mean(tr_rewards)}\nAverage of the total actions: {numpy.mean(tr_actions)}')

#Testing
env.reset()
rewards = []
actions = []
for i in range(10):
    state = env.reset()
    done = False
    total_rewards = 0
    total_actions = 0
    for j in range(25):
        env.render()

        action = numpy.argmax(Q_table[state])
        total_actions += 1

        new_state, reward, done, info = env.step(action)
        total_rewards += reward

        if done:
            rewards.append(total_rewards)
            actions.append(total_actions)
            break
        state = new_state
print(f'Average of total reward: {numpy.mean(rewards)}\nAverage of the total actions: {numpy.mean(actions)}')
env.close()


from multiprocessing import Pool
from functools import partial
import Libs.create_fold as create_fold
import Libs.file_storage as file_handle
import config

class Run_model(object):
  experiment_type = None
  run_func = None
  run_parallel = False
  feature_type = 'tfidf'

  def __init__(self, run_func, experiment_type, run_parallel = False):
    self.run_func = run_func
    self.experiment_type = experiment_type
    self.run_parallel = run_parallel

  def run_tm(self):
    self.feature_type = 'tm'

    print('running TM')
    print('running a total of %i datasets' % config.NUM_TOPICMODELS)

    for dataset_num in range(1, config.NUM_TOPICMODELS + 1):
      topicmodel = file_handle.load_topicmodel(dataset_num)
      tm_matrix = topicmodel['matrix']

      self.run_folds(tm_matrix, dataset_num)

  def run_tf(self):
    self.feature_type = 'tfidf'

    print('running TF-IDF')
    tf_matrix = file_handle.load_matrix('tfidf_matrix')

    self.run_folds(tf_matrix)

  def run_folds(self, matrix, dataset_num = 1):
    fold_creator = create_fold.Create_fold(matrix, self.experiment_type)

    num_folds = fold_creator.determine_folds()
    print('running a total of %i folds' % num_folds)

    fold_range = range(1, num_folds + 1)

    if self.run_parallel:
      # Fix parameters of run_fold function
      parallel_func = partial(self.run_fold,
          fold_creator = fold_creator,
          run_func = self.run_func,
          feature_type = self.feature_type,
          dataset_num = dataset_num,
          experiment_type = self.experiment_type)

      with Pool(processes = config.POOL_PROCESSES) as pool:
        # Parallelise the folds
        pool.map(parallel_func, fold_range)
    else:
      for fold in fold_range:
        self.run_fold(fold, fold_creator, self.run_func, self.feature_type, dataset_num, self.experiment_type)

    print('Number of docs in matrix: %i' % fold_creator.X.shape[0])
    print('Number of docs in test sets: %i' % fold_creator.x_test_total_size)

  def run_fold(self, fold, fold_creator, run_func, feature_type, dataset_num, experiment_type):
    print('on fold #%i' % fold)

    classifier_obj = file_handle.load_classifier(fold, feature_type, dataset_num, experiment_type)

    if classifier_obj is None:
      data = fold_creator.get_fold(fold)

      classifier_obj = run_func(data, fold)

      file_handle.store_classifier(fold, feature_type, classifier_obj, dataset_num, experiment_type)


import os
import sys
import tensorflow.compat.v1 as tf
import glob
from lxml import etree
import io
import PIL.Image

from object_detection.utils import dataset_util
from object_detection.utils import label_map_util

def dict_to_tfrecord(img_path, data):
    with tf.gfile.GFile(img_path, 'rb') as fid:
        encoded_jpg = fid.read()
    encoded_jpg_io = io.BytesIO(encoded_jpg)
    image = PIL.Image.open(encoded_jpg_io)

    if image.format != 'JPEG':
        raise ValueError('Image format not JPEG')

    width = int(data['size']['width'])
    height = int(data['size']['height'])

    xmin = []
    ymin = []
    xmax = []
    ymax = []
    classes = []
    classes_text = []

    if 'object' in data:
        for obj in data['object']:
            difficult = bool(int(obj['difficult']))

            xmin.append(float(obj['bndbox']['xmin']) / width)
            ymin.append(float(obj['bndbox']['ymin']) / height)
            xmax.append(float(obj['bndbox']['xmax']) / width)
            ymax.append(float(obj['bndbox']['ymax']) / height)
            classes_text.append(obj['name'].encode('utf8'))
            classes.append(label_map_dict[obj['name']])

    tf_example = tf.train.Example(features=tf.train.Features(feature={
        'image/height': dataset_util.int64_feature(height),
        'image/width': dataset_util.int64_feature(width),
        'image/filename': dataset_util.bytes_feature(
            data['filename'].encode('utf8')),
        'image/source_id': dataset_util.bytes_feature(
            data['filename'].encode('utf8')),
        'image/encoded': dataset_util.bytes_feature(encoded_jpg),
        'image/format': dataset_util.bytes_feature('jpeg'.encode('utf8')),
        'image/object/bbox/xmin': dataset_util.float_list_feature(xmin),
        'image/object/bbox/xmax': dataset_util.float_list_feature(xmax),
        'image/object/bbox/ymin': dataset_util.float_list_feature(ymin),
        'image/object/bbox/ymax': dataset_util.float_list_feature(ymax),
        'image/object/class/text': dataset_util.bytes_list_feature(classes_text),
        'image/object/class/label': dataset_util.int64_list_feature(classes),
    }))

    return tf_example

if __name__ == '__main__':
    print("Starting Train Record")
    writer = tf.python_io.TFRecordWriter('./annotations/train.record')
    label_map_dict = label_map_util.get_label_map_dict('./annotations/label_map.pbtxt')
    os.chdir("./images/train/")
    for file in glob.glob("*.xml"):
        file_name = file.split('.')[:-1]
        file_name = '.'.join(file_name)

        with tf.gfile.GFile(file, 'r') as fid:
            xml_str = fid.read()

        xml = etree.fromstring(xml_str)
        data = dataset_util.recursive_parse_xml_to_dict(xml)['annotation']

        tf_example = dict_to_tfrecord(os.path.abspath(f'{file_name}.jpg'), data)
        writer.write(tf_example.SerializeToString())
    writer.close()

    os.chdir("../../")

    print("Starting Test Record")
    writer = tf.python_io.TFRecordWriter('./annotations/test.record')
    os.chdir("./images/test/")
    for file in glob.glob("*.xml"):
        file_name = file.split('.')[:-1]
        file_name = '.'.join(file_name)

        with tf.gfile.GFile(file, 'r') as fid:
            xml_str = fid.read()

        xml = etree.fromstring(xml_str)
        data = dataset_util.recursive_parse_xml_to_dict(xml)['annotation']

        tf_example = dict_to_tfrecord(os.path.abspath(f'{file_name}.jpg'), data)
        writer.write(tf_example.SerializeToString())
    writer.close()


"""modules for pytroch-lighting cifar10"""
# pylint: disable=W0201
import os
import pickle
from typing import Optional
from PIL import Image
import pytorch_lightning as pl
from torch.utils.data import DataLoader, Dataset
import torchvision
from torchvision import transforms
import numpy as np

from config import CFG
from utils import mysplit

stats = ((0.4914, 0.4822, 0.4465), (0.2022, 0.1994, 0.2009))

def prepare_imgs_and_targets(data_dir, train=True):
    """_summary_

    Args:
        data_dir (str): path of data directory
        train (bool, optional): whether to collect data for train or not. Defaults to True.

    Returns:
        imgs, traget labels(10classes for cifar10)
    """
    train_list = [
        "data_batch_1",
        "data_batch_2",
        "data_batch_3",
        "data_batch_4",
        "data_batch_5",
    ]
    test_list = ["test_batch"]
    imgs = []
    targets = []
    if train:
        downloaded_list = train_list
    else:
        downloaded_list = test_list
    for file_name in downloaded_list:
        file_path = os.path.join(data_dir, file_name)
        with open(file_path, "rb") as file:
            mydict = pickle.load(file, encoding="bytes")
            this_batch_img = (
                mydict[b"data"].reshape((-1, 3, 32, 32)).transpose(0, 2, 3, 1)
            )
            this_batch_targets = mydict[b"labels"]
            if file_name == "data_batch_1" or train is False:
                imgs = this_batch_img
                targets = this_batch_targets
            else:
                imgs = np.concatenate([imgs, this_batch_img], axis=0)
                targets = np.concatenate([targets, this_batch_targets], axis=0)

    return imgs, targets

class MyDataset(Dataset):
    """Classic Dataset with img and target

    Args:
        Dataset (_type_): Cifar10 dataset
        if using albumentation, use the part that is annotated
    """

    def __init__(self, imgs, targets, transform):
        self.imgs = imgs
        self.targets = targets
        self.transform = transform

    def __getitem__(self, index):
        img, target = self.imgs[index], self.targets[index]
        img = Image.fromarray(img)
        img = self.transform(img)
        # if self.transform is not None:
        #     img = self.transform(image=img)["image"]
        return img, target

    def __len__(self):
        return len(self.imgs)

class CIFAR10Data(pl.LightningDataModule):
    """Pytorch lightning cifar10 datamodule

    Args:
        root(str): path of data directory
        batch_size(int): batch size
        this_fold(int): fold number
        transform(callable, optional): transform function. Defaults to None.
    """

    def __init__(
        self,
        root: str,
        batch_size: int,
        this_fold: int,
        transform: Optional[callable] = None,
    ) -> None:
        super().__init__()
        self.root = root
        self.batch_size = batch_size
        self.this_fold = this_fold
        self.transform = transform

    def prepare_data(self) -> None:
        pass  # already downloaded

    def setup(self, stage: Optional[str] = None) -> None:
        if stage == "fit":
            image, target = prepare_imgs_and_targets(self.root, train=True)
            img_train, img_valid, target_train, target_valid = mysplit(
                image, target, self.this_fold, CFG.n_split, CFG.seed
            )
            self.trainset = MyDataset(
                img_train,
                target_train,
                transform=transforms.Compose(
                    [
                        torchvision.transforms.RandomCrop(32, padding=4),
                        torchvision.transforms.RandomHorizontalFlip(),
                        torchvision.transforms.ToTensor(),
                        torchvision.transforms.Normalize(*stats),
                    ]
                ),
            )
            self.validset = MyDataset(
                img_valid,
                target_valid,
                transform=transforms.Compose(
                    [
                        torchvision.transforms.ToTensor(),
                        torchvision.transforms.Normalize(*stats),
                    ]
                ),
            )
        if stage == "test":
            image, target = prepare_imgs_and_targets(self.root, train=False)
            self.testset = MyDataset(
                image,
                target,
                transform=transforms.Compose(
                    [
                        torchvision.transforms.ToTensor(),
                        torchvision.transforms.Normalize(*stats),
                    ]
                ),
            )

    def train_dataloader(self) -> DataLoader:
        cifar_train = DataLoader(
            self.trainset,
            batch_size=self.batch_size,
            shuffle=True,
            num_workers=CFG.num_workers,
            drop_last=False,
        )
        return cifar_train

    def val_dataloader(self) -> DataLoader:
        cifar_valid = DataLoader(
            self.validset,
            batch_size=10 * self.batch_size,
            shuffle=False,
            num_workers=CFG.num_workers,
            drop_last=False,
        )
        return cifar_valid

    def test_dataloader(self) -> DataLoader:
        cifar_test = DataLoader(
            self.testset,
            batch_size=10 * self.batch_size,
            shuffle=False,
            num_workers=CFG.num_workers,
            drop_last=False,
        )
        return cifar_test


import vlc
from time import sleep
import json
import os

import voice_info

class iRadio:
    def __init__(self, config_file: str, station_file: str):

        self.init(config_file, station_file)

        if self.state == 'on':
            self.play()

    def init(self, config_file: str, station_file: str):
        print("iRadio init..");

        self.config_file = config_file
        self.station_file = station_file
        self.instance = vlc.Instance()
        self.player = self.instance.media_player_new()

        self.state = "off"
        self.current_station = 0

        with open(self.config_file) as c:
            self.config_dict = json.load(c)
            p = self.config_dict['radio_config']
            self.state = p['state']
            self.current_station = p['current_station']

        if self.state == "off":
            print("iRadio state: OFF")
        elif self.state == "on":
            print("iRadio state: ON")

        self.stations = None

        with open(self.station_file) as s:
            station_dict = json.load(s)
            self.stations = station_dict['stations']
            self.station_count = len(self.stations)

        print("Loaded \'" + str(self.station_count) + "\' stations..")
        print("Current station: " + str(self.stations[self.current_station]["name"]))
        print("\nDone!")

    def get_station_name(self):
        return self.stations[self.current_station]["name"]

    def get_station_pronounced_name(self):
        return self.stations[self.current_station]["pronounce"]

    def is_playing(self):
        if self.player.is_playing() == 1:
            return True
        else:
            return False

    def say_something(self, something):
        sleep(2)
        self.player.audio_set_volume(60)
        sleep(1)

        print(voice_info.say_text(something))

        sleep(5)
        self.player.audio_set_volume(100)

    def play(self):
        Media = self.instance.media_new(self.stations[self.current_station]["url"])
        Media.get_mrl()
        self.player.set_media(Media)

        if self.player.play() == -1:
            print ("Error playing Stream")
            self.state = 'off'
            return False

        self.say_something("Posloucháte %s" % self.get_station_pronounced_name())

        self.state = 'on'

        # save config
        p = self.config_dict['radio_config']
        p['state'] = self.state
        self.save_config()
        return True

    def stop(self):
        self.player.stop()

        print("Stop..")
        self.state = 'off'

        # save config
        p = self.config_dict['radio_config']
        p['state'] = self.state
        self.save_config()

    def next(self):
        self.stop()

        if self.current_station == (self.station_count - 1):
            self.current_station = 0
        else:
            self.current_station += 1

        if not self.play():
            sleep(2)
            self.next()
            return
        #self.play()

        # save config
        p = self.config_dict['radio_config']
        p['current_station'] = self.current_station
        self.save_config()

    def prev(self):
        self.stop()

        if self.current_station == 0:
            self.current_station = self.station_count - 1
        else:
            self.current_station -= 1

        if not self.play():
            sleep(2)
            self.prev()
            return
        #self.play()

        # save config
        p = self.config_dict['radio_config']
        p['current_station'] = self.current_station
        self.save_config()

    def save_config(self):
        # save config file
        os.remove(self.config_file)
        with open(self.config_file, 'w') as c:
            json.dump(self.config_dict, c)

if __name__ == "__main__":

    radio = iRadio("config_radio.json", "radio_stations.json")
    #radio.play()

    while True:
        sleep(60)
        radio.next()


from collections import namedtuple
from functools import wraps
from packaging import version

import torch
from torch import nn, einsum
import torch.nn.functional as F

from einops import rearrange

# constants

EfficientAttentionConfig = namedtuple('EfficientAttentionConfig', ['enable_flash', 'enable_math', 'enable_mem_efficient'])

# helpers

def exists(val):
    return val is not None

def once(fn):
    called = False
    @wraps(fn)
    def inner(x):
        nonlocal called
        if called:
            return
        called = True
        return fn(x)
    return inner

print_once = once(print)

# main class

class Attend(nn.Module):
    def __init__(
        self,
        causal = False,
        dropout = 0.,
        flash = False
    ):
        super().__init__()
        self.dropout = dropout
        self.attn_dropout = nn.Dropout(dropout)

        self.causal = causal
        self.flash = flash
        assert not (flash and version.parse(torch.__version__) < version.parse('2.0.0')), 'in order to use flash attention, you must be using pytorch 2.0 or above'

        # determine efficient attention configs for cuda and cpu

        self.cpu_config = EfficientAttentionConfig(True, True, True)
        self.cuda_config = None

        if not torch.cuda.is_available() or not flash:
            return

        device_properties = torch.cuda.get_device_properties(torch.device('cuda'))

        if device_properties.major == 8 and device_properties.minor == 0:
            print_once('A100 GPU detected, using flash attention if input tensor is on cuda')
            self.cuda_config = EfficientAttentionConfig(True, False, False)
        else:
            print_once('Non-A100 GPU detected, using math or mem efficient attention if input tensor is on cuda')
            self.cuda_config = EfficientAttentionConfig(False, True, True)

    def get_mask(self, i, j, device):
        return torch.ones((i, j), device=device, dtype=torch.bool).triu(j - i + 1)

    def flash_attn(self, q, k, v, mask = None, attn_bias = None):
        _, heads, q_len, _, k_len, is_cuda, device = *q.shape, k.shape[-2], q.is_cuda, q.device

        # single headed key / values

        if k.ndim == 3:
            k = rearrange(k, 'b n d -> b 1 n d')

        if v.ndim == 3:
            v = rearrange(v, 'b n d -> b 1 n d')

        # Check if mask exists and expand to compatible shape
        # The mask is B L, so it would have to be expanded to B H N L

        if exists(mask) and mask.ndim != 4:
            mask = rearrange(mask, 'b j -> b 1 1 j')
            mask = mask.expand(-1, heads, q_len, -1)

        # Check if there is a compatible device for flash attention

        config = self.cuda_config if is_cuda else self.cpu_config

        causal = self.causal

        # handle attention bias

        if exists(attn_bias):
            mask_value = -torch.finfo(q.dtype).max // 2
            causal_mask = self.get_mask(q_len, k_len, device)
            attn_bias = attn_bias.masked_fill(causal_mask, mask_value)

            if exists(mask):
                attn_bias = attn_bias.masked_fill(~mask, mask_value)

            mask = attn_bias
            causal = False

        # pytorch 2.0 flash attn: q, k, v, mask, dropout, causal, softmax_scale

        with torch.backends.cuda.sdp_kernel(**config._asdict()):
            out = F.scaled_dot_product_attention(
                q, k, v,
                attn_mask = mask,
                dropout_p = self.dropout if self.training else 0.,
                is_causal = causal
            )

        return out

    def forward(self, q, k, v, mask = None, attn_bias = None):
        """
        einstein notation
        b - batch
        h - heads
        n, i, j - sequence length (base sequence length, source, target)
        d - feature dimension
        """

        q_len, k_len, device = q.shape[-2], k.shape[-2], q.device

        scale = q.shape[-1] ** -0.5

        kv_einsum_eq = 'b j d' if k.ndim == 3 else 'b h j d'

        if self.flash:
            assert not exists(attn_bias)
            return self.flash_attn(q, k, v, mask = mask)

        # similarity

        sim = einsum(f"b h i d, {kv_einsum_eq} -> b h i j", q, k) * scale

        # attention bias

        if exists(attn_bias):
            sim = sim + attn_bias

        # causal mask

        if self.causal:
            causal_mask = self.get_mask(q_len, k_len, device)
            sim = sim.masked_fill(causal_mask, -torch.finfo(sim.dtype).max)

        # key padding mask

        if exists(mask):
            if mask.ndim != 4:
                mask = rearrange(mask, 'b j -> b 1 1 j')
            sim = sim.masked_fill(~mask, -torch.finfo(sim.dtype).max)

        # attention

        attn = sim.softmax(dim=-1)
        attn = self.attn_dropout(attn)

        # aggregate values

        out = einsum(f"b h i j, {kv_einsum_eq} -> b h i d", attn, v)

        return out


import pandas
from sklearn.linear_model import LogisticRegression
from sklearn.neural_network import MLPClassifier
from sklearn.tree import DecisionTreeClassifier

print(__doc__)

# Modified for MCZA015-13 class project by Rodrigo Martins de Oliveira
# License: BSD Style.

import matplotlib.pyplot as plt
import numpy as np
from scipy import interp
from sklearn.ensemble import ExtraTreesClassifier as SKExtraTreesClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import roc_curve, auc
from sklearn.model_selection import StratifiedKFold

from extra_trees.ensemble.forest import ExtraTreesClassifier

# #############################################################################
# Data IO and generation

# Import some data to play with
# iris = datasets.load_iris()
# X = iris.data
# y = iris.target
# X, y = X[y != 2], y[y != 2]
# n_samples, n_features = X.shape
#
# # Add noisy features
# random_state = np.random.RandomState(0)
# X = np.c_[X, random_state.randn(n_samples, 200 * n_features)]
url = "https://archive.ics.uci.edu/ml/machine-learning-databases/pima-indians-diabetes/pima-indians-diabetes.data"
names = ['preg', 'plas', 'pres', 'skin', 'test', 'mass', 'pedi', 'age', 'class']
dataframe = pandas.read_csv(url, names=names)
array = dataframe.values
X = array[:, 0:8]
y = array[:, 8]

# #############################################################################
# Classification and ROC analysis

# Run classifier with cross-validation and plot ROC curves
cv = StratifiedKFold(n_splits=6)

# prepare models
classification_models = [
    ('Logistic', LogisticRegression()),
    ('Neural Net', MLPClassifier()),
    ('DecisionTree', DecisionTreeClassifier()),
    ('RandomForest', RandomForestClassifier()),
    ('ExtraTrees (SciKit)', SKExtraTreesClassifier()),
    ('ExtraTrees', ExtraTreesClassifier()),
]

folds = list(cv.split(X, y))

plt.plot([0, 1], [0, 1], linestyle='--', lw=2, color='r',
         label='Luck', alpha=.8)

for name, classifier in classification_models:
    tprs = []
    aucs = []
    mean_fpr = np.linspace(0, 1, 100)

    for train, test in folds:
        probas_ = classifier.fit(X[train], y[train]).predict_proba(X[test])
        # Compute ROC curve and area the curve
        fpr, tpr, thresholds = roc_curve(y[test], probas_[:, 1])
        tprs.append(interp(mean_fpr, fpr, tpr))
        tprs[-1][0] = 0.0
        roc_auc = auc(fpr, tpr)
        aucs.append(roc_auc)

    mean_tpr = np.mean(tprs, axis=0)
    mean_tpr[-1] = 1.0
    mean_auc = auc(mean_fpr, mean_tpr)
    std_auc = np.std(aucs)
    plt.plot(mean_fpr, mean_tpr, "-",
             label=r'%s (AUC = %0.2f $\pm$ %0.2f)' % (name, mean_auc, std_auc),
             lw=2, alpha=.8)

plt.xlim([-0.05, 1.05])
plt.ylim([-0.05, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic example')
plt.legend(loc="lower right")
plt.show()


from flask import Flask, render_template, request
from flask import jsonify
import configparser
import sqlite3
import random
from process_db import main
import load_xml

app = Flask(__name__)

# Database
DATABASE = 'kTunes.sqlite'

def get_db_connection():
    conn = sqlite3.connect(DATABASE)
    conn.row_factory = sqlite3.Row
    return conn

# Define the path to the config file
config_path = "ktunes.ini"
config = configparser.ConfigParser()
config.read(config_path)
if not config.has_section("misc"):
    print("config doesn't have misc section, gonna write out a default one")
    default_config = {
        "misc": {
            "playlist_name": "",
            "playlist_lgth": "",
            "recently_added_switch": "",
            "RA_weighting_factor": "",
            "RA_play_cnt": "",
            "create_playlist": "",
            "debug_lvl": ""
        },
        "categories": {
            "cat1": 'zed',
            "cat1_pct": '33',
            "cat1_repeat": 33
        }
    }
    print("before reading default_config")
    config.read_dict(default_config)
    with open(config_path, "w") as config_file:
        config.write(config_file)
else:
    print("config has misc section:",config["misc"])

# List of background images
background_images = [
    'PiecesOfEight1.jpg',
    'PiecesOfEight2.jpg',
    'PiecesOfEight3.jpg',
    'PiecesOfEight4.jpg',
    'PiecesOfEight5.jpg',
    ]
# Select a random background image from the list
selected_image = random.choice(background_images)

# Generate the URL for the selected image
background_image_url = f"/static/images/{selected_image}"

# Function to query a table with filters and paginate the result set
def query_table(**kwargs):
    db = get_db_connection()
    count_query = f"SELECT count(*) FROM ({kwargs['query']})"
    #print("count_query=",count_query)
    cursor = db.execute(count_query)
    count = cursor.fetchone()[0]
    offset = (kwargs['page'] - 1) * kwargs['per_page']
    limit = kwargs['per_page']
    order_by_str = ', '.join(kwargs['order_by_cols'])
    query = f"{kwargs['query']} ORDER BY {order_by_str}  LIMIT ?, ?"
    cursor = db.execute(query, (offset, limit))
    tracks = cursor.fetchall()
    cursor.close()
    return tracks, count

@app.route("/config", methods=["GET", "POST"])
def create_playlist():
    # Read the configuration file
    config.read(config_path)

    if request.method == "POST":    # Update the configuration file with the new values
            # Read the category percentage values from the form
        form_data = request.form
        # display each field and its value
        for field, value in form_data.items():
            print(f"{field}: {value}")
        pcts = [
        request.form["cat1_pct"],
        request.form["cat2_pct"],
        request.form["cat3_pct"],
        request.form["cat4_pct"],
        request.form["cat5_pct"]
       ]

        # Convert the percentage values to floats and compute the total
        total_pct = sum(float(pct) for pct in pcts)

        # Check if the total is 100, and if not, display an error message
        if total_pct != 100:
            error_msg = "# # # Category percentages must add up to 100 (current total is {}). Values reset. # # #.".format(total_pct)
            return render_template("config.html", config=config, error=error_msg)

        config["misc"]["playlist_name"] = request.form["playlist_name"]
        config["misc"]["playlist_lgth"] = request.form["playlist_lgth"]
        if "recently_added_switch" in request.form:
          config["misc"]["recently_added_switch"] = request.form["recently_added_switch"]
        else:
          config["misc"]["recently_added_switch"] = 'no'
        config["misc"]["RA_weighting_factor"] = request.form["RA_weighting_factor"]
        config["misc"]["RA_play_cnt"] = request.form["RA_play_cnt"]
        config["misc"]["create_playlist"] = request.form["create_playlist"]
        config["misc"]["debug_lvl"] = request.form["debug_lvl"]

        config.set("categories", "cat1", request.form["cat1"])
        config.set("categories", "cat1_pct", request.form["cat1_pct"])
        config.set("categories", "cat1_repeat", request.form["cat1_repeat"])

        config.set("categories", "cat2", request.form["cat2"])
        config.set("categories", "cat2_pct", request.form["cat2_pct"])
        config.set("categories", "cat2_repeat", request.form["cat2_repeat"])

        config.set("categories", "cat3", request.form["cat3"])
        config.set("categories", "cat3_pct", request.form["cat3_pct"])
        config.set("categories", "cat3_repeat", request.form["cat3_repeat"])

        config.set("categories", "cat4", request.form["cat4"])
        config.set("categories", "cat4_pct", request.form["cat4_pct"])
        config.set("categories", "cat4_repeat", request.form["cat4_repeat"])

        config.set("categories", "cat5", request.form["cat5"])
        config.set("categories", "cat5_pct", request.form["cat5_pct"])
        config.set("categories", "cat5_repeat", request.form["cat5_repeat"])

        # Write the updated configuration file
        with open(config_path, "w") as config_file:
            config.write(config_file)
        config_parser_dict = {s:dict(config.items(s)) for s in config.sections()}
        total_songs, nbr_of_genre_songs, dup_playlist = main(config_parser_dict)
    # Read the configuration file
    config.read(config_path)

    # Render the template with the configuration data
    return render_template("config.html", config=config)

# Route to display the paginated table of playlist
@app.route('/playlist', methods=['GET', 'POST'])
def playlist():
    q_dict = {}
    screen_height = request.args.get('screen_height', type=int)
    if screen_height:
        print("got a screen_height of",screen_height)
        max_rows_response = requests.get(f"{request.url_root}get_max_rows?screen_height={screen_height}")
        if max_rows_response.status_code == 200:
            max_rows = max_rows_response.json().get('max_rows')
            per_page = max_rows
        else:
            per_page = 20
    else:
        print("didn't get a screen height")
        per_page = request.args.get('per_page', 20, type=int)

    try:
        page = int(request.args.get('page', 1))
    except ValueError:
        page = 1

    q_dict['playlist'] = request.args.get('playlist', 'kTunes_03')
    q_dict['artist'] = request.args.get('artist', '')
    q_dict['song'] = request.args.get('song', '')
    q_dict['category'] = request.args.get('category', '')

    query = "SELECT playlist_nm, track_cnt, artist, song, category, play_cnt, strftime('%Y-%m-%d', last_play_dt) AS last_play_dt FROM playlist_tracks WHERE lower(playlist_nm) like '" + q_dict['playlist'].lower() + "%'"
    if q_dict['artist']:
        query += " AND lower(Artist) LIKE '%" + q_dict['artist'].lower() + "%'"
    if q_dict['song']:
        query += " AND lower(song) LIKE '%" + q_dict['song'].lower() +  "%'"
    if q_dict['category']:
        query += " AND category LIKE '%" + q_dict['category'].lower() + "%'"

    q_dict['query'] = query
    q_dict['order_by_cols'] = ['track_cnt']
    q_dict['per_page'] = per_page
    q_dict['page'] = page

    tracks, count = query_table(**q_dict)
    total_pages = (count + per_page - 1) // per_page
    start_page = max(1, page - 2)
    end_page = min(total_pages, page + 2)
    endpoint = 'playlist'
    template_name= 'playlist.html'
    print('background image=',background_image_url)
    return render_template('table_templ.html', template_name=template_name, endpoint=endpoint, tracks=tracks, count=count, total_pages=total_pages, page=page, per_page=per_page, start_page=start_page, end_page=end_page, q_dict=q_dict)#, background_image_url=background_image_url)

# Route to display the paginated table of tracks
@app.route('/tracks', methods=['GET', 'POST'])
def tracks():
    q_dict = {}
    screen_height = request.args.get('screen_height', type=int)
    if screen_height:
        print("got a screen_height of",screen_height)
        max_rows_response = requests.get(f"{request.url_root}get_max_rows?screen_height={screen_height}")
        if max_rows_response.status_code == 200:
            max_rows = max_rows_response.json().get('max_rows')
            per_page = max_rows
        else:
            per_page = 20
    else:
        print("didn't get a screen height!")
        per_page = request.args.get('per_page', 20, type=int)

    try:
        page = int(request.args.get('page', 1))
    except ValueError:
        page = 1

    q_dict['artist'] = request.args.get('artist', '')
    q_dict['song'] = request.args.get('song', '')
    q_dict['category'] = request.args.get('category', '')

    query = "SELECT Artist, song, category, Album, play_cnt, strftime('%Y-%m-%d', last_play_dt) AS last_play_dt, strftime('%Y-%m-%d', date_added) AS date_added FROM tracks WHERE 1=1"
    if q_dict['artist']:
        query += " AND lower(Artist) LIKE '%" + q_dict['artist'].lower() + "%'"
    if q_dict['song']:
        query += " AND lower(song) LIKE '%" + q_dict['song'].lower() +  "%'"
    if q_dict['category']:
        query += " AND category LIKE '%" + q_dict['category'].lower() + "%'"

    q_dict['query'] = query
    q_dict['order_by_cols'] = ['artist', 'song']
    q_dict['per_page'] = per_page
    q_dict['page'] = page

    tracks, count = query_table(**q_dict)
    total_pages = (count + per_page - 1) // per_page
    start_page = max(1, page - 2)
    end_page = min(total_pages, page + 2)
    template_name= 'tracks.html'
    endpoint='tracks'
    print('background image=',background_image_url)
    return render_template('table_templ.html', template_name=template_name, endpoint=endpoint, tracks=tracks, count=count, total_pages=total_pages, page=page, per_page=per_page, start_page=start_page, end_page=end_page, q_dict=q_dict, background_image_url=background_image_url)

if __name__ == "__main__":
   app.run(debug=True)


from typing import Iterator, Optional

import pdfplumber
import polars as pl

def get_page_table_rows(page: pdfplumber.page.Page) -> list[list[Optional[str]]]:
    tables = page.extract_tables()
    return tables[0] if tables else []

def get_all_table_rows(
    pdf: pdfplumber.pdf.PDF, page_indices: list[int]
) -> Iterator[list[Optional[str]]]:
    for page_ix in page_indices:
        page = pdf.pages[page_ix]
        rows = get_page_table_rows(page)
        yield from rows

COLUMNS = [
    "grant_id",
    "provider_name",
    "responses_total",
    "responses_top_2",
    "performance_rating",
]

def main() -> None:
    pdf = pdfplumber.open("pdfs/VA SSVF Year End Report FY20.pdf")
    page_ixs = [20, 21, 22, 23, 24]
    rows = list(get_all_table_rows(pdf, page_ixs))

    (
        pl.DataFrame(rows[1:], schema=COLUMNS)
        .filter(pl.col("grant_id") != "Grant ID")
        .with_columns(pl.col("performance_rating").str.strip("%").cast(float))
        .write_csv("data/from-pdfs/provider-satisfaction-fy20.csv")
    )

if __name__ == "__main__":
    main()


import train_functions as tf
from kernels import kernel_list
import numpy as np
from beta_train import beta_train
from draw import draw_pred_difference
from sklearn.kernel_ridge import KernelRidge

class MTMK:
    # 初始化
    # 调整Kernel直接调整kernel_list，这里不放kernel的参数的接口了
    def __init__(self,kernel_list:list=kernel_list(),task_num=4) -> None:
        # 任务数量
        self.task_num = task_num

        self.kernel_list = kernel_list
        # self.alpha = alpha
        # self.lmd = lmd  # lambda是正则化的参数

        # 需要拟合的参数
        self.eta = []
        self.beta = []

        # 参数的Loss值
        self.loss_eta = []
        self.loss_beta = []

        # 训练集的数据
        self.train_data = []
        pass

    def fit(self,X_train,Y_train,t0,mu,alpha,lmd,tol,epoch_num) -> None:

        self.train_data = X_train

        # 训练eta的代码
        # self.eta, self.loss_eta, AK = tf.eta_train(
        #     KernelFunctionList=self.kernel_list,
        #     DataX=X_train,DataY=Y_train,
        #     t0=t0,
        #     mu=mu,
        #     lmd=lmd,
        #     tol=tol,
        #     alpha=alpha,
        #     Epoch_num=epoch_num)

        # 强行把每个task的eta设置为[1,0,0,0]，测试beta的功能
        for i in range(0,self.task_num):
            tmp = np.zeros(shape=(4,1),dtype=float)
            tmp[0] = 1
            self.eta.append(tmp)

        print("MTMK eta training finished")

        self.beta, self.loss_beta = beta_train(
            KernelFunctionList=self.kernel_list,
            DataX=X_train,DataY=Y_train,
            eta=self.eta,alpha=alpha,lmd=lmd,Epoch_num=epoch_num)
        # print(self.beta)
        print("MTMK beta training finished")

        pass

    # 预测，给出预测的结果，连续值或者离散的0/1形式的label
    # 给出某个任务的预测：
    def predict_single_task(self,task_order:int,X_test_k,continuous:bool=True)->list:
        # task_order = k
        k = task_order
        # eta_k = self.eta[k]
        beta_k = self.beta[k]
        yk_pred_list = []

        x_train_k = self.train_data[k]
        N_t = x_train_k.shape[0]
        # 遍历samples
        for i in range(X_test_k.shape[0]):
            x = X_test_k[i]
            ret = 0
            # 遍历样本
            for j in range(N_t):
                xj = x_train_k[j]
                ret += beta_k[j]*tf.kernel_combined(
                    self.kernel_list,self.eta,k,x,xj)
            # for j in range(N_t):
            y = 1/(1+np.exp(-ret))

            # 是否离散/连续化预测：
            if continuous: yk_pred_list.append(y)
            else:
                if y > 0.5 : yk_pred_list.append(1)
                else: yk_pred_list.append(0)

        return np.array(yk_pred_list).reshape(-1,1)

    def predict_all_task(self,X_test:list,continuous:bool=True) ->list:
        ret = []
        # for x_test_k in X_test:
        for i in range(0,len(X_test)):
            ret.append(self.predict_single_task(
                task_order=i, X_test_k=X_test[i],
                continuous=continuous
            ))
        return ret

# 调试MTMK
if __name__ == "__main__":
    X_data = []
    Y_data = []
    SAMPLES = 100
    # 选择了100个samples，1个task，1维的数据，效果很差
    X_data.append(np.random.uniform(low=0,high=1,size=SAMPLES).reshape(-1,1))
    Y_data.append(1/(1+np.exp(-X_data[0])))

    # 很关键一步，将函数变成列表类型
    kernels = kernel_list()
    model = MTMK(kernel_list=kernels,task_num=1)
    model.fit(X_data,Y_data,t0=1,mu=1.1,alpha=0.001,lmd=1,tol=0.0001,epoch_num=5000)
    y_pred = model.predict_all_task(X_data,True)

    model = KernelRidge(alpha=1,kernel="linear")
    model.fit(X_data[0],Y_data[0])
    y_pred_krr = model.predict(X_data[0].reshape(-1,1))

    # print(f"predicted label:{y_pred}")
    # print(f"real label:{Y_data}")

    # 对比随机数据集里，KRR，MTMK，真实数据集的不同
    draw_pred_difference(n_samples=SAMPLES,y_real=Y_data[0],
    y_mtmk=y_pred[0],y_krr=y_pred_krr)
    pass


import numpy as np
from DrumsDataset_vae_genre import DrumsDataset
import torch
from torch.utils.data.dataset import random_split
import torch.optim as optim
import torch.nn as nn
from DNnet import DNnet

local_dataset='/home/ftamagna/Documents/_AcademiaSinica/dataset/drumGeneration/FillsExtractedSupervised_cleaned_v2.npz'

class TrainingGenerator:

    def __init__(self,batch_size,lr,n_epochs,dataset_filepath=local_dataset):

        self.dataset_filepath=dataset_filepath
        self.batch_size=batch_size
        self.lr=lr
        self.n_epochs=n_epochs

        self.use_cuda = torch.cuda.is_available()
        device = torch.device("cuda" if self.use_cuda else "cpu")

        if self.use_cuda:
            print('run on GPU')
        else:
            print('run on CPU')

    def load_data(self,binarize=True):
        data = np.load(self.dataset_filepath)
        data=dict(data)
        vae=data['vae']
        genre=data['genre']
        self.dataset=DrumsDataset(numpy_array=vae,genre=genre,use_cuda=self.use_cuda)
        print(len(self.dataset),"LEN DATASET")
    def split_data(self):
        ratio = 0.6
        train_length = int(len(self.dataset) * ratio)
        validation_length = int((len(self.dataset) - train_length) * 0.5)
        test_length = len(self.dataset) - validation_length - train_length
        self.train, self.validation, self.test = random_split(self.dataset,
                                                              [train_length, validation_length, test_length])

        self.train_loader = torch.utils.data.DataLoader(dataset=self.train, batch_size=self.batch_size, shuffle=True,drop_last=True)
        self.test_loader = torch.utils.data.DataLoader(dataset=self.test, batch_size=self.batch_size, shuffle=False,drop_last=False)
        self.validation_loader = torch.utils.data.DataLoader(dataset=self.test, batch_size=self.batch_size,
                                                             shuffle=False,drop_last=True)

    def train_model(self):
        if self.use_cuda:
            dnn=DNnet().cuda()
        else:
            dnn=DNnet()

        criterion = nn.MSELoss()
        optimizer = optim.SGD(dnn.parameters(), lr=self.lr, momentum=0.9,weight_decay=0.6)

        for epoch in range(self.n_epochs):  # loop over the dataset multiple times

            running_loss = 0.0
            for i, data in enumerate(self.train_loader, 0):
                # get the inputs
                inputs,g, labels = data

                # zero the parameter gradients
                optimizer.zero_grad()

                # forward + backward + optimize
                outputs = dnn(inputs,g)
                loss = criterion(outputs, labels)
                loss.backward()
                optimizer.step()

                # print statistics
                running_loss += loss.item()
                # if i % 20 == 0:  # print every 2000 mini-batches
            print('[%d, %5d] loss: %.10f' %
                  (epoch + 1, i + 1, running_loss / len(self.train)))
            running_loss = 0.0
            with torch.no_grad():
                total_val_loss = 0
                for i, data in enumerate(self.test_loader, 0):
                    inputs, g,labels = data
                    val_outputs = dnn(inputs,g)
                    val_loss_size = criterion(val_outputs, labels)
                    total_val_loss += val_loss_size.item()
                print('Val loss: %.10f' %(total_val_loss / (i+1)))
                # print(len(self.test))

        print('Finished Training')

        self.net = dnn

    def save_model(self, filepath, name):
        torch.save(self.net.state_dict(), filepath + name)

if __name__=="__main__":

    LR=0.001
    BATCH_SIZE=600
    N_EPOCHS=100

    tg=TrainingGenerator(lr=LR,batch_size=BATCH_SIZE,n_epochs=N_EPOCHS)
    tg.load_data()
    tg.split_data()
    tg.train_model()
    tg.save_model("./../models/",'vae_generation__c_cleaned_v2.pt')


from art import logo
import random

EASY_LEVEL_TURNS = 10
HARD_LEVEL_TURNS = 5

#Function to check user's guess against actual answer.
def check_answer(guess, answer, turns):
    """checks answer against guess. Returns the number of turns remaining."""
    if guess > answer:
        print("Too high")
        return turns - 1
    elif guess < answer:
        print("Too low")
        return turns - 1
    elif guess == answer:
        print(f"You got it! The answer was {answer}")

#Make function to set difficulty.
def set_difficulty():
    """Returns the number of attempts based on the choice of difficulty"""
    difficulty = ''
    while difficulty != 'easy' and difficulty != 'hard':
        difficulty = input("Choose a difficulty. Type 'easy' or 'hard': ").lower()
        if difficulty == 'easy':
            attemps = EASY_LEVEL_TURNS
        elif difficulty == 'hard':
            attemps = HARD_LEVEL_TURNS
        else:
            print("The mode isn't accept")
    return attemps

def guessthenumber():
    """The game"""
    print(logo)
    print("Welcome to the Number Guessing Game!")
    print("I'm thinking of a number between 1 and 100.")
    random_number = random.randint(1, 100)
    attemps = set_difficulty()
    guess_number = 0
    while guess_number != random_number:
        print(f"You have {attemps} attemps remaining to guess the number.")
        guess_number = int(input("Make a guess: "))
        attemps = check_answer(guess_number, random_number, attemps)
        if attemps == 0:
            print("You've run out of guesses, you lose")
            break
        elif guess_number != random_number:
            print("Guess again")

guessthenumber()


""" matman <github.com/fiedlr/matman> """

import csv
import sys
import math
import codecs
import statistics

def update_avg(avg, nvals, new_val):
	""" Updates an already existing average """
	return (avg + (new_val / nvals)) * (nvals / (nvals + 1)) if nvals else new_val

def open_csv(filename):
	""" Opens a csv file in utf8 and returns a csvr object """
	file = codecs.open(filename, encoding='utf8')
	# Decode the file
	reader = csv.reader(file)
	# Skip the header
	next(reader)
	# Return the reader object
	return reader

class Analyser:
	""" The scanner class """

	# STATSAMPLE = {'nanswers': 0, 'ncorrect': 0, 'avgresp': 0, 'avgerr': 0}

	def __init__(self, skillfile, itemfile, datafile):
		# Too many attributes

		# Save the filenames
		self.__skillfile = skillfile
		self.__itemfile = itemfile
		self.__datafile = datafile
		# Entries counter
		self.__count_entries = {'all': 0, 'fake': 0, 'invalid': 0}
		# Create the analysis structures
		self.__items = {}
		self.__students = {}
		self.__skills = {
			# Counting
			'1870': {
				'1871': {'label': 'numbers_leq_10', 'count': 0, 'avgdiff': 0},
				'1872': {'label': 'numbers_leq_20', 'count': 0, 'avgdiff': 0}
			},
			# Addition
			'1893': {
				'1894': {'label': 'addition_leq_10', 'count': 0, 'avgdiff': 0},
				'1895': {'label': 'addition_leq_20', 'count': 0, 'avgdiff': 0},
				'1896': {'label': 'addition_leq_100', 'count': 0, 'avgdiff': 0}
			},
			# Subtraction
			'2018': {
				'2019': {'label': 'subtraction_leq_10', 'count': 0, 'avgdiff': 0},
				'2020': {'label': 'subtraction_leq_20', 'count': 0, 'avgdiff': 0},
				# arbitrary
				'2021': {'label': 'subtraction_leq_100', 'count': 0, 'avgdiff': 0}
			},
			# Multiplication
			'2076': {
				'2077': {'label': 'multiplication_small', 'count': 0, 'avgdiff': 0},
				'2133': {'label': 'multiplication_big', 'count': 0, 'avgdiff': 0}
			},
			# Division
			'2234': {
				'2235': {'label': 'division1', 'count': 0, 'avgdiff': 0}
			}
		}
		# Stat containers
		self.__success_sd = 0 # success standard deviation
		self.__est = {'fakers': 0, 'underavg': 0, 'overavg': 0} # estimates
		self.__savg = {'success': 0, 'resp': 0, 'err': 0} # student general avgs
		self.__skillstat = {}
		self.__visualstat = {'written_question': [0, 0], 'object_counting': [0, 0], 'object_counting_with_numbers': [0, 0], 'object_selection_answer': [0, 0], 'number_line_answer': [0, 0], 'multiplication_visualization_field': [0, 0], 'division_visualization_baskets': [0, 0]} # stats about the visual types
		self.__time_conversion = [] # avg. error to avg. time
		self.__difficulty_list = []
		# Set a higher limit for the csv reader
		csv.field_size_limit(sys.maxsize)

	def build(self):
		""" Create a temp db for storing item defs """
		csvr = open_csv(self.__itemfile)
		# Process the output
		for entry in csvr:
			item = str(entry[0]); visual = str(entry[3]); skill = str(entry[4]); skills = (str(entry[5]), str(entry[6]), str(entry[7])); question = str(entry[1])
			self.__items[item] = {'nanswers': 0, 'ncorrect': 0, 'avgresp': 0, 'medresp': 0, 'avgerr': 0, 'resps': [], 'visual': visual, 'skill': skill, 'skills': skills, 'question': question}

	def run(self):
		""" Runs through the dataset """
		# Open the file in UTF8 to avoid errors
		csvr = open_csv(self.__datafile)
		# Process the output
		for entry in csvr:
			# Increase the counters
			self.__count_entries['all'] += 1
			# Filter out irrelevant answers
			try:
				answer = int(entry[6])
			except ValueError:
				self.__count_entries['invalid'] += 1
				continue
		    # Any answer greater than 500 is surely a scam
			if answer > 500:
				self.__count_entries['fake'] += 1
				continue
		    # Basic conversions
			item = str(entry[2]); student = str(entry[3]); time = float(entry[4]); correct = int(entry[5]); expected = int(entry[7]);

			# Calculate error
			dev = abs(answer - expected)

			# Recalculate item averages
			self.__items[item]['avgresp'] = (
				update_avg(self.__items[item]['avgresp'], self.__items[item]['nanswers'], time)
			)
			self.__items[item]['avgerr'] = (
				update_avg(self.__items[item]['avgerr'], self.__items[item]['nanswers'], dev)
			)
			# Store for median calculation
			self.__items[item]['resps'].append(time)
			# Items counter
			self.__items[item]['ncorrect'] += correct
			self.__items[item]['nanswers'] += 1

			# Recalculate student averages
			if student not in self.__students:
				# Add student to "database"
				self.__students[student] = {'nanswers': 0, 'ncorrect': 0, 'avgresp': 0, 'avgerr': 0}
			else:
				self.__students[student]['avgresp'] = (
					update_avg(self.__students[student]['avgresp'], self.__students[student]['nanswers'], time)
				)
				self.__students[student]['avgerr'] = (
					update_avg(self.__students[student]['avgerr'], self.__students[student]['nanswers'], dev)
				)
			# Students counter
			self.__students[student]['ncorrect'] += correct
			self.__students[student]['nanswers'] += 1

	def __guess_faker(self, avgresp, avgerr):
		""" Estimate faker status against the avgs """
		return avgresp < self.__savg['resp'] and avgerr > self.__savg['err']

	def __guess_underavg(self, avgresp, avgerr, srate):
		""" Estimates underavg status against the avgs """
		return avgresp < self.__savg['resp'] and avgerr > self.__savg['err'] and srate < self.__savg['success']

	def __guess_overavg(self, avgresp, avgerr, srate):
		""" Estimates overavg status against the avgs """
		return avgresp > self.__savg['resp'] and avgerr < self.__savg['err'] and srate > self.__savg['success']

	def analyse_students(self):
		""" Runs through the results and makes the appropriate averages """
		nstudents = len(self.__students)
		for s in self.__students.values():
			srate = float(s['ncorrect']) / s['nanswers']
			#if s['avgresp'] <= 2000:
			self.__time_conversion.append((s['avgresp'], s['avgerr']))
			self.__savg['success'] += srate
			self.__savg['resp'] += s['avgresp']
			self.__savg['err'] += s['avgerr']
		# Calculate averages
		self.__savg['success'] /= nstudents
		self.__savg['resp'] /= nstudents
		self.__savg['err'] /= nstudents
		# Calculate success standard deviation
		standard_dev = 0
		# Estimate the possible number of fakers, overavgs and underavgs
		for s in self.__students.values():
			srate = float(s['ncorrect']) / s['nanswers']
			standard_dev += (srate - self.__savg['success']) ** 2
			self.__est['fakers'] += int(self.__guess_faker(s['avgresp'], s['avgerr']))
			self.__est['overavg'] += int(self.__guess_overavg(s['avgresp'], s['avgerr'], srate))
			self.__est['underavg'] += int(self.__guess_underavg(s['avgresp'], s['avgerr'], srate))
		self.__success_sd = math.sqrt(standard_dev / nstudents)
		return [self.__savg['resp'], self.__savg['success'], self.__savg['err'], self.__success_sd, self.__est['fakers'], self.__est['underavg'], self.__est['overavg']]

	def analyse_items(self):
		""" Go through items, categorize them and analyse them accordingly to their difficulty """
		for key, i in self.__items.items():
			# Calculate the median
			i['medresp'] = statistics.median(i['resps'])
			# Calculate difficulty
			difficulty = round((i['avgerr'] * i['medresp']) / (i['ncorrect'] / i['nanswers']), 2)
			i['difficulty'] = difficulty
			# Output for analysis
			self.__difficulty_list.append((key, difficulty))
			# Group difficulties to skills
			skill = str(i['skills'][0]); skillcat = str(i['skills'][1])
			# Gaps
			if skill == skillcat == 2018 or len(skillcat) == 0:
				skillcat = '2021'
			self.__skills[skill][skillcat]['avgdiff'] = (
				update_avg(self.__skills[skill][skillcat]['avgdiff'], self.__skills[skill][skillcat]['count'], difficulty)
			)
			self.__skills[skill][skillcat]['count'] += 1
			# Group itemstats to visuals
			self.__visualstat[i['visual']][0] += 1
			self.__visualstat[i['visual']][1] += i['difficulty']
		# Calculate averages for visualizations
		for visualavg in self.__visualstat.values():
			visualavg[1] /= visualavg[0]
		self.__difficulty_list = sorted(self.__difficulty_list, key=lambda item: item[1], reverse=True)
		return self.__difficulty_list

	def analyse_skills(self):
		""" Sort difficulties in the skill levels for analysis """
		skillstat = {}
		for key, skill in self.__skills.items():
			if key not in skillstat:
				skillstat[key] = []
			# Output for plotting
			for i, level in skill.items():
				skillstat[key].append((i, level['avgdiff']))
			# Sort in an ascending order
			skillstat[key] = sorted(skillstat[key], key=lambda item: item[1])
		# Save to state
		self.__skillstat = skillstat
		return skillstat

	def get_skills(self):
		""" Get the skills processed dataset """
		return self.__skills

	def get_items(self):
		""" Get the items processed dataset """
		return self.__items

	def get_students(self):
		""" Get the students processed dataset """
		return self.__students

	def get_num_entries(self):
		""" Puts the results in a meaningful data object for Plotter """
		return self.__count_entries

	def time_conversion(self):
		""" Return avg time to avg error conversion """
		return self.__time_conversion

	def difficulty_list(self):
		""" Get the items sorted by their difficulty """
		return self.__difficulty_list

	def visualstat(self):
		""" Visualization statistics """
		return self.__visualstat

	def skillstat(self):
		""" Get the skill statistics """
		return self.__skillstat


# 장하다...풀었다..

from itertools import permutations
from copy import deepcopy

def calcul(a, b, c):
    if c == '*':
        return a * b
    elif c == '-':
        return a - b
    elif c == '+':
        return a + b

def solution(expression):
    answers = []

    expression += '-' # 숫자와 연산자 구분을 위해 그냥 끝에 - 붙여줌. 밑에 for문 보면암.
    answer = 0
    oper = ['*', '+', '-']
    orders = list(permutations(oper, 3))
    print('!', orders)
    solid_nums = []
    solid_chars = []

    # 숫자와 연산자 나눠서 배열에 각각 넣음.
    tmp = ''
    for i in expression:
        if i.isdigit():
            tmp += i
        else:
            solid_nums.append(int(tmp))
            tmp = ''
            solid_chars.append(i)

    del solid_chars[-1] # 17번째줄에 넣어준 "-" 는 삭제함.
    print('solid_nums', solid_nums)
    print('chars', solid_chars)

    for p in range(len(orders)):
        order = orders[p]
        nums = deepcopy(solid_nums) # 카피해야 for문 돌때 정상적으로 원하는 계산이됨.
        chars = deepcopy(solid_chars) # 여기도.

        for i in range(3):
            print(nums)
            turn = order[i]
            for j in range(len(chars)):
                if chars[j] == turn:
                    nums[j+1] = calcul(nums[j], nums[j+1], turn)
                    nums[j] = 'X'
                    chars[j] = 'X'
            for k in range(len(chars)-1, -1, -1):
                if nums[k] == 'X':
                    del nums[k]
                if chars[k] == 'X':
                    del chars[k]

        print(nums)
        answers.append(nums[0])

    answers = [abs(i) for i in answers]
    return answers

print(solution("100-200*300-500+20"))

카카오문제/2020인턴쉽/수식최대화.py


import aoc
from collections import Counter

def inpfct(x):
    return list(map(int, list(x)))

inp = aoc.get_input(inpfct)

_logger = aoc.get_logger(__name__)

width=25
height=6
n = width*height

layers = [inp[i:i+n] for i in range(0, len(inp), n)]
if aoc.part_one():
    _logger.info('%s Layers found (size: %s x %s)', len(layers), width, height)
    clayers = sorted(list(map(Counter, layers)), key=lambda x: x[0])
    print(clayers[0][1]*clayers[0][2])

if aoc.part_two():
    print()
    cnt = 0
    layer = [2] * n
    for l in layers:
        for i, y in enumerate(l):
            if layer[i] == 2:
                layer[i] = y
    for y in range(0, height):
        for x in range(0, width):
            if layer[y*width+x] == 1:
                print('X', end='')
            else:
                print(' ', end='')
        print()


import sys
from optparse import OptionParser
from time import sleep

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation

import pymoduleconnector
from pymoduleconnector import DataType
from pathlib import Path
import serial
import serial.tools.list_ports

import warnings
warnings.filterwarnings("ignore")

# Current directory for data recording storage
currentPath = str(Path().absolute())
storePath = currentPath + '\\' + 'recordedData\\'

# Reset module method
def resetModule(devicePort):
    mc = pymoduleconnector.ModuleConnector(devicePort)
    x4m03 = mc.get_xep()
    x4m03.module_reset()
    mc.close()
    sleep(3)

def clearBuffer(mc):
    """Clears the frame buffer"""
    x4m03 = mc.get_xep()
    while x4m03.peek_message_data_float():
        x4m03.read_message_data_float()

def simple_xep_plot(devicePort, record=False, baseband=True, startDist=0, endDist=5):

    FPS = 20
    directory = '.'
    resetModule(devicePort)
    mc = pymoduleconnector.ModuleConnector(devicePort)

    # Driver parameter initialization
    x4m03 = mc.get_xep()
    # Set DAC range
    x4m03.x4driver_set_dac_min(900)
    x4m03.x4driver_set_dac_max(1150)

    # Set integration
    x4m03.x4driver_set_iterations(16)
    x4m03.x4driver_set_pulses_per_step(26)

    x4m03.x4driver_set_downconversion(int(baseband))

    # Edit radar range
    # Frame offset for X4M03 is fixed on 0.18 meters
    x4m03.x4driver_set_frame_area_offset(0.18)
    x4m03.x4driver_set_frame_area(startDist, endDist)

    # Start streaming of data
    x4m03.x4driver_set_fps(FPS)

    def readFrame():
        """Gets frame data from module"""
        d = x4m03.read_message_data_float()
        frame = np.array(d.data)
         # Convert the resulting frame to a complex array if downconversion is
         # enabled
        if baseband:
            n = len(frame)
            frame = frame[:n//2] + 1j*frame[n//2:]
        return frame

    quit = False

    plt.ion()
    plt.figure()
    plt.title("X4M03 Baseband Data")

    clearBuffer(mc)
    frame = readFrame()

    t = np.arange(len(frame))

    data_amp = np.array(t)
    data_phi = np.array(t)

    while True:

        clearBuffer(mc)
        frame = readFrame()
        amps = np.abs(frame)
        phis = np.angle(frame)

        plt.subplot(211)
        plt.title("X4M03 Baseband Data")
        plt.subplots_adjust(hspace=0.4)

        plt.ylim((-0.001, 0.03))
        plt.plot(amps)
        plt.xlabel("Bin #")
        plt.ylabel("Normalized Amplitude")
        plt.pause(0.001)
        plt.grid(True)

        plt.subplot(212)
        plt.ylim((-3.5, 3.5))
        plt.plot(phis)
        plt.xlabel("Bin #")
        plt.ylabel("Phase [rad]")
        plt.pause(0.001)
        plt.grid(True)

        data_amp = np.vstack((data_amp, np.abs(frame)))
        data_phi = np.vstack((data_phi, np.angle(frame)))

        while True:
            print("Comands: q - quit, s - new record, Enter - new data, \
                   w name - save data to file")
            command = input("Enter command >> ")
            print("Command: ",command)

            if (command == "Q" or command == "q"):
                quit = True
                break
            elif(command == "s" or command == ""):
                plt.clf()
                data_amp = np.array(t)
                data_phi = np.array(t)
                break
            elif(command == "c"):
                break
            elif(command[0] == "w" and command[1]==" "):
                filename_amp = storePath + command[2:] + "_amp.csv"
                filename_phi = storePath + command[2:] + "_phi.csv"
                np.savetxt(filename_amp, np.transpose(data_amp), delimiter=", ")
                print("File recorded with name: ", filename_amp)
                np.savetxt(filename_phi, np.transpose(data_phi), delimiter=", ")
                print("File recorded with name: ", filename_phi)

        if (quit):
            break

    x4m03.x4driver_set_fps(0)
    print("Exiting...")

def main():
    # Code block for automatic search of port where X4M03 is connected
    devicePort = ''
    print('Searching...')
    ports = serial.tools.list_ports.comports(include_links=False)
    for port in ports :
        print('Found port '+ port.device)
        devicePort = port.device

    startDist = 0
    endDist = 5
    simple_xep_plot(devicePort, record=False,
                                baseband=True,
                                startDist=0,
                                endDist=5)

if __name__ == "__main__":
   main()


import sys
import networkx as nx
nodes = set()
edges = []
with open(sys.argv[1], "r") as f:
    for line in f:
        raw = line.rstrip().split(' ')
        current = int(raw.pop(0))
        raw.pop(0) # <->
        nodes.add(current)
        while len(raw):
            edges.append((current, int(raw.pop(0).replace(',', ''))))
G = nx.Graph()
G.add_nodes_from(nodes)
G.add_edges_from(edges)
print(len(nx.node_connected_component(G, 0)))
print(sum([1 for component in nx.connected_components(G)]))


"""请求方式的封装"""
import requests
from Apiframework.common.yaml_util import YamlUtil

class Request_util:

    sess = requests.session()

    # 通用请求方式
    def all_send_requests(self, method, url, **kwargs):
        res = Request_util.sess.request(method, url, **kwargs)
        return res

# a = Request_util()
# url = YamlUtil().read_test_yaml("url3", "../data/url.yaml")
# res = a.post_api(url, data=YamlUtil().read_test_yaml("url3_data", "../data/url.yaml"))
# print(res)


from datetime import datetime
from odoo import api, fields, models, _
from odoo.exceptions import ValidationError as UserError

class HrTimesheetSheet(models.Model):
    _inherit = "hr_timesheet_sheet.sheet"

    @api.model
    def _default_date_from(self):
        res = super(HrTimesheetSheet, self)._default_date_from()
        period = self._get_current_pay_period()
        if period:
            return period.date_start
        else:
            return res

    @api.model
    def _default_date_to(self):
        res = super(HrTimesheetSheet, self)._default_date_to()
        period = self._get_current_pay_period()
        if period:
            return period.date_end
        else:
            return res

    @api.model
    def _default_hr_period_id(self):
        return self._get_current_pay_period()

    hr_period_id = fields.Many2one('hr.period', string='Pay Period',
                                   readonly=True,
                                   states={'new': [('readonly', False)]},
                                   default=_default_hr_period_id)
    date_from = fields.Date('Date from', required=True, index=True,
                            readonly=True,
                            states={'new': [('readonly', False)]},
                            default=_default_date_from)
    date_to = fields.Date('Date to', required=True, index=True, readonly=True,
                          states={'new': [('readonly', False)]},
                          default=_default_date_to)

    @api.multi
    def name_get(self):
        if not self._ids:
            return []
        if isinstance(self._ids, (long, int)):
            self._ids = [self._ids]
        res = super(HrTimesheetSheet, self).name_get()
        res2 = []
        for record in res:
            sheet = self.browse(record[0])
            if sheet.hr_period_id:
                record = list(record)
                name = sheet.hr_period_id.name
                record[1] = name
                res2.append(tuple(record))
            else:
                res2.append(record)
        return res2

    @api.multi
    @api.onchange('hr_period_id')
    def onchange_pay_period(self):
        if self.hr_period_id:
            self.date_from = self.hr_period_id.date_start
            self.date_to = self.hr_period_id.date_end
            self.name = self.hr_period_id.name

    @api.model
    def _get_current_pay_period(self):
        period_obj = self.env['hr.period']
        contract_obj = self.env['hr.contract']
        date_today = datetime.today().strftime('%Y-%m-%d')
        employee = self.default_get(['employee_id'])
        contract = contract_obj.search([('employee_id', '=',
                                         employee.get('employee_id'))])
        search_domain = [('date_start', '<=', date_today),
                         ('date_end', '>=', date_today)]
        if contract and contract.schedule_pay:
            search_domain += [('schedule_pay', '=', contract.schedule_pay)]
        period_ids = period_obj.search(search_domain)
        if period_ids:
            return period_ids[0]
        else:
            return False

    @api.multi
    @api.constrains('date_from', 'date_to', 'hr_period_id')
    def _check_start_end_dates(self):
        for timesheet in self:
            if timesheet.hr_period_id:
                if timesheet.date_from != timesheet.hr_period_id.date_start:
                    raise UserError(
                        _("The Date From of Timesheet must match with that of"
                          " date start '%s' of the Payroll period '%s'.") % (
                            timesheet.hr_period_id.date_start,
                            timesheet.hr_period_id.name))
                if timesheet.date_to != timesheet.hr_period_id.date_end:
                    raise UserError(
                        _("The Date To of Timesheet must match with that of"
                          " date stop '%s' of the Payroll period '%s'.") % (
                            timesheet.hr_period_id.date_end,
                            timesheet.hr_period_id.name))


"""Handling Starmade Blueprint name searching in App Engine"""

from google.appengine.api import search
from datetime import datetime

_INDEX_NAME = "blueprints"

def create_document(title, blue_key):
    """Creates a search.Document from the blueprint name."""

    # Let the search service supply the document id.
    doc = search.Document(
        fields=[search.TextField(name='title', value=title),
                search.TextField(name='blue_key', value=blue_key),
                search.DateField(name='date', value=datetime.now().date())])

    search.Index(name=_INDEX_NAME).put(doc)

    return None


from panda import Panda
import json

class PandaAlreadyThere(Exception):
    pass

class PandasAlreadyFriends(Exception):
    pass

class PandaSocialNetwork:

    def __init__(self):
        self.network = {}

    def has_panda(self, panda):
        if panda in self.network:
            return True
        return False

    def __getitem__(self, index):
        return self.network[index]

    def add_panda(self, panda):
        if self.has_panda(panda):
            raise PandaAlreadyThere

        self.network[panda] = []

    def are_friends(self, panda1, panda2):
        if not self.has_panda(panda1) or not self.has_panda(panda2):
            return False

        if panda1 in self.network[panda2] and panda2 in self.network[panda1]:
            return True
        return False

    def make_friends(self, panda1, panda2):
        if not self.has_panda(panda1):
            self.add_panda(panda1)

        if not self.has_panda(panda2):
            self.add_panda(panda2)

        if self.are_friends(panda1, panda2):
            raise PandasAlreadyFriends

        self.network[panda1].append(panda2)
        self.network[panda2].append(panda1)

    def friends_of(self, panda):
        if panda not in self.network.keys():
            return False
        return self.network[panda]

    def connection_level(self, panda1, panda2):
        if self.are_friends(panda1, panda2):
            return 1
        if not self.has_panda(panda1) or not self.has_panda(panda2):
            return False
        if self.bfs(panda1, panda2) == 0:
            return -1
        else:
            return self.bfs(panda1, panda2)

    def bfs(self, panda1, panda2):
        visited = set()
        queue = []
        path_to = {}

        queue.append(panda1)
        visited.add(panda1)
        path_to[panda1] = None
        found = False
        path_length = 0

        while len(queue) != 0:
            current_node = queue.pop(0)
            if current_node == panda2:
                found = True
                break
            for neighbour in self.network[current_node]:
                if neighbour not in visited:
                    path_to[neighbour] = current_node
                    visited.add(neighbour)
                    queue.append(neighbour)

        if found:
            while path_to[panda2] is not None:
                path_length += 1
                panda2 = path_to[panda2]

        return path_length

    def are_connected(self, panda1, panda2):
        if self.connection_level(panda1, panda2) > 0:
            return True
        else:
            return False

    def how_many_gender_in_network(self, level, panda, gender):
        gender_count = 0

        for friend in self.network:
            current_level = self.connection_level(panda, friend)
            if friend != panda and current_level <= level and \
                    friend.get_gender() == gender and current_level != -1:
                gender_count += 1

        return gender_count

    def save(self, path):
        pandas_data = {}
        for panda in self.network:
            pandas_data[repr(panda)] = []
            for friend in self.network[panda]:
                pandas_data[repr(panda)] += [repr(friend)]

        json_string = json.dumps(pandas_data, indent=4)
        with open(path, "w") as save_file:
            save_file.write(json_string)

    def load(self, path):
        contents = {}
        with open(path, "r") as load_file:
            contents = json.loads(load_file.read())

        for panda in contents.keys():
            for friend in contents[panda]:
                if not self.are_friends(panda, friend):
                    self.make_friends(panda, friend)

        return self

def main():
    ivo = Panda("ivo", "ivo@gmail.com", "male")
    rado = Panda("rado", "radorado@mail.bg", "male")
    mary = Panda("mary", "mary@abv.bg", "female")
    tony = Panda("tony", "tony@mail.bg", "female")
    network = PandaSocialNetwork()
    network.add_panda(ivo)
    network.add_panda(rado)
    network.add_panda(mary)
    network.make_friends(ivo, rado)
    network.make_friends(rado, mary)
    network.make_friends(ivo, tony)
    network.save("panda_social_network.json")

    new_network = PandaSocialNetwork()
    new_network.load("panda_social_network.json")
    print (new_network.network)

if __name__ == '__main__':
    main()