#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Target visibility plotting.
@TODO: Move to flows-tools.

.. codeauthor:: Rasmus Handberg <rasmush@phys.au.dk>
"""

import logging
import os.path
import warnings
from datetime import datetime
from typing import Optional

import astropy.units as u
import matplotlib.pyplot as plt
import numpy as np
from astropy.coordinates import AltAz, SkyCoord, get_moon, get_sun
from astropy.time import Time
from astropy.visualization import quantity_support
from matplotlib.dates import DateFormatter
from tendrils import api

from .target import Target

# --------------------------------------------------------------------------------------------------
def visibility(target: Target, siteid: Optional[int] = None, date=None, output=None, overwrite=True):
    """
    Create visibility plot.

    Parameters:
        target (str or int):
        siteid (int): Identifier of site.
        date (datetime or str, optional): Date for which to create visibility plot.
            Default it to use the current date.
        output (str, optional): Path to file or directory where to place the plot.
            If not given, the plot will be created in memory, and can be shown on screen.
        overwrite (bool, optional): Should existing file specified in ``output`` be overwritten?
            Default is to overwrite an existing file.

    .. codeauthor:: Rasmus Handberg <rasmush@phys.au.dk>
    """

    logger = logging.getLogger(__name__)
    warnings.warn(DeprecationWarning("This module is moved to SNFLOWS/flows-tools."))

    if date is None:
        date = datetime.utcnow()
    elif isinstance(date, str):
        date = datetime.strptime(date, '%Y-%m-%d')

    # Coordinates of object:
    obj = SkyCoord(ra=target.ra, dec=target.dec, unit='deg', frame='icrs')

    if siteid is None:
        sites = api.get_all_sites()
    else:
        sites = [api.get_site(siteid)]

    plotpaths = []
    for site in sites:
        # If we are saving plot to file, determine the path to save to
        # and check that it doesn't already exist:
        if output:
            if os.path.isdir(output):
                plotpath = os.path.join(output, "visibility_%s_%s_site%02d.png" % (
                    target.name, date.strftime('%Y%m%d'), site['siteid']))
            else:
                plotpath = output
            logger.debug("Will save visibility plot to '%s'", plotpath)

            # If we are not overwriting and
            if not overwrite and os.path.exists(plotpath):
                logger.info("File already exists: %s", plotpath)
                continue
        logger.debug(f"{site}, type: {type(site)}")
        # Observatory:
        observatory = site.get('EarthLocation', None)
        if observatory is None:
            observatory = site.get('earth_location')
        utcoffset = (site['longitude'] * u.deg / (360 * u.deg)) * 24 * u.hour

        # Create timestamps to calculate for:
        midnight = Time(date.strftime('%Y-%m-%d') + ' 00:00:00', scale='utc') - utcoffset
        delta_midnight = np.linspace(-12, 12, 1000) * u.hour
        times = midnight + delta_midnight

        # AltAz frame:
        AltAzFrame = AltAz(obstime=times, location=observatory)

        # Object:
        altaz_obj = obj.transform_to(AltAzFrame)

        # The Sun and Moon:
        altaz_sun = get_sun(times).transform_to(AltAzFrame)
        altaz_moon = get_moon(times).transform_to(AltAzFrame)

        sundown_astro = (altaz_sun.alt < -6 * u.deg)
        if np.any(sundown_astro):
            min_time = np.min(times[sundown_astro]) - 2 * u.hour
            max_time = np.max(times[sundown_astro]) + 2 * u.hour
        else:
            min_time = times[0]
            max_time = times[-1]

        quantity_support()
        fig, ax = plt.subplots(1, 1, figsize=(15, 9), squeeze=True)
        plt.grid(ls=':', lw=0.5)
        ax.plot(times.datetime, altaz_sun.alt, color='y', label='Sun')
        ax.plot(times.datetime, altaz_moon.alt, color=[0.75] * 3, ls='--', label='Moon')
        objsc = ax.scatter(times.datetime, altaz_obj.alt, c=altaz_obj.az, label=target.name, lw=0, s=8,
                           cmap='twilight')
        ax.fill_between(times.datetime, 0 * u.deg, 90 * u.deg, altaz_sun.alt < -0 * u.deg, color='0.5',
                        zorder=0)  # , label='Night'
        ax.fill_between(times.datetime, 0 * u.deg, 90 * u.deg, altaz_sun.alt < -18 * u.deg, color='k',
                        zorder=0)  # , label='Astronomical Night'

        plt.colorbar(objsc, ax=ax, pad=0.01).set_label('Azimuth [deg]')
        ax.legend(loc='upper left')
        ax.minorticks_on()
        ax.set_xlim(min_time.datetime, max_time.datetime)
        ax.set_ylim(0 * u.deg, 90 * u.deg)
        ax.set_title("%s - %s - %s" % (target.name, date.strftime('%Y-%m-%d'), site['sitename']),
                     fontsize=14)
        plt.xlabel('Time [UTC]', fontsize=14)
        plt.ylabel('Altitude [deg]', fontsize=16)
        fig.autofmt_xdate()

        formatter = DateFormatter('%d/%m %H:%M')
        ax.xaxis.set_major_formatter(formatter)

        if output:
            fig.savefig(plotpath, bbox_inches='tight', transparent=True)
            plt.close(fig)
            plotpaths.append(plotpath)

    if output:
        return plotpaths

    plt.show()
    return ax


import io
from unittest import TestCase
from unittest.mock import patch
from books import move_book_no_result_retry
from book import Book

class TestMoveBookNoResultRetry(TestCase):

    @patch('books.search_book', return_value=[Book("123", "April C", "Hacking with April", "Chris", "9", "science",
                                                   "computer")])
    @patch('sys.stdout', new_callable=io.StringIO)
    def test_move_book_no_result_retry_wrong_input_string(self, mock_output, mock_search_book):
        book_1 = Book("123", "April C", "Hacking with April", "Chris", 9, "science", "computer")
        book_2 = Book("456", "Lipra", "Coding with April", "Johnson", 10, "math", "matrix")
        filtered_list = []
        book_collection = [book_1, book_2]
        move_book_no_result_retry(filtered_list, book_collection)
        expected = 'There are no results, please retry so we can move the book after.\n\n'
        self.assertEqual(mock_output.getvalue(), expected)

    @patch('books.search_book')
    def test_move_book_no_result_retry_wrong_search_book_called(self, mock_search_book):
        book_1 = Book("123", "April C", "Hacking with April", "Chris", 9, "science", "computer")
        book_2 = Book("456", "Lipra", "Coding with April", "Johnson", 10, "math", "matrix")
        filtered_list = []
        book_collection = [book_1, book_2]
        move_book_no_result_retry(filtered_list, book_collection)
        mock_search_book.assert_called_with(book_collection)

    @patch('books.search_book')
    def test_move_book_no_result_retry_correct_input(self, mock_search_book):
        book_1 = Book("123", "April C", "Hacking with April", "Chris", 9, "science", "computer")
        book_2 = Book("456", "Lipra", "Coding with April", "Johnson", 10, "math", "matrix")
        filtered_list = [book_2]
        book_collection = [book_1, book_2]
        actual = move_book_no_result_retry(filtered_list, book_collection)
        self.assertEqual(actual, filtered_list)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import sys
from collections import defaultdict

from Bio.PDB import NeighborSearch, PDBParser, Selection

import lt

POSITIVE_BOND_CUTOFF = 4.0

def pdb_neighbors(pdb_f, pdb_id):
    structure = PDBParser().get_structure(pdb_id, pdb_f)
    atom_list = Selection.unfold_entities(structure, 'A')
    ns = NeighborSearch(atom_list)
    center_res = [res for res in structure.get_residues() if res.get_resname() in ['PTR','SEP','TPO']]

    neighbors = []
    for res in center_res:
        if res.get_resname() == 'PTR':
            central_atoms = [atom for atom in res.child_list if atom.get_name() in ['O1P', 'O2P', 'O3P', 'OH']]
        elif res.get_resname() == 'SEP':
            central_atoms = [atom for atom in res.child_list if atom.get_name() in ['O1P', 'O2P', 'O3P', 'OG']]
        elif res.get_resname() == 'TPO':
            central_atoms = [atom for atom in res.child_list if atom.get_name() in ['O1P', 'O2P', 'O3P', 'OG1']]

        positive_atom_neighbors = [ns.search(a.get_coord(),POSITIVE_BOND_CUTOFF) for a in central_atoms]
        positive_atom_neighbors = [atom for atoms in positive_atom_neighbors for atom in atoms]
        positive_atom_neighbors = [atom for atom in positive_atom_neighbors if atom.get_name() in ['NE2','ND1','NZ','NE','NH2','NH1']]

        atom_neighbors = positive_atom_neighbors
        atom_neighbors = list(set(atom_neighbors))

        atom_neighbors = list(set(Selection.unfold_entities(atom_neighbors,'R')))
        atom_neighbors = [r for r in atom_neighbors if r != res]

        if len(atom_neighbors) > 0:
            res = res.get_resname()+'_'+str(res.get_id()[1])+'_'+res.get_parent().get_id()
            atom_neighbors = [n.get_resname()+'_'+str(n.get_id()[1])+'_'+n.get_parent().get_id()+'_'+'S' for n in atom_neighbors]
            neighbors.append((pdb_id,res,atom_neighbors))

    return neighbors

def main():
    res_neighbors = []
    for pdb_f in lt.files_in_dir(sys.argv[-1]):
        f_path,f_name = os.path.split(pdb_f)
        f_name,f_exten = os.path.splitext(f_name)
        if f_exten == '.pdb':
            pdb_id = f_name
            res_neighbors.extend(pdb_neighbors(pdb_f, pdb_id))

    with lt.open_file(file_name='salt_bridge_initial') as w_f:
        for pro, phos_res, neighbor_res in res_neighbors:
            print >> w_f, '{0:<8}{1:<15}{2}'.format(
                pro, phos_res, ',   '.join(neighbor_res))

    lt.pickle_dump(res_neighbors,'salt_bridge')

if __name__ == "__main__":
    main()


import matplotlib.pyplot as plt
import gym
from agent import MyLovelyAgent, EpsilonGreedyParameters
from off_policy_teacher import MrAdamsTheTeacher
import torch
import torchvision.transforms.functional as vision_f
from torchvision.transforms import transforms as tr
from datetime import datetime
from experience import ExperienceBuffer, Experience
import numpy as np
from typing import Union
from PIL import Image

import os
os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"

def plot(rewards, mean_values):
    plt.clf()
    plt.title("Training!!!!")
    plt.xlabel("Episodes")
    plt.ylabel("Reward")
    plt.plot(rewards)
    plt.plot(mean_values)
    plt.pause(0.001)

def make_state_image(env: gym.Env, transforms) -> torch.Tensor:
    # Returned screen requested by gym is 400x600x3, but is sometimes larger
    # such as 800x1200x3. Transpose it into torch order (CHW).
    screen = env.render(mode='rgb_array').transpose((2, 0, 1))
    # Cart is in the lower half, so strip off the top and bottom of the screen
    _, screen_height, screen_width = screen.shape
    screen = screen[:, int(screen_height*0.4):int(screen_height * 0.8)]
    view_width = int(screen_width * 0.6)
    cart_location = get_cart_location(screen_width)
    if cart_location < view_width // 2:
        slice_range = slice(view_width)
    elif cart_location > (screen_width - view_width // 2):
        slice_range = slice(-view_width, None)
    else:
        slice_range = slice(cart_location - view_width // 2,
                            cart_location + view_width // 2)
    # Strip off the edges, so that we have a square image centered on a cart
    screen = screen[:, :, slice_range]
    # Convert to float, rescale, convert to torch tensor
    # (this doesn't require a copy)
    screen = np.ascontiguousarray(screen, dtype=np.float32) / 255
    screen = torch.from_numpy(screen)
    # Resize, and add a batch dimension (BCHW)
    return transforms(screen)

def get_cart_location(screen_width):
    world_width = env.x_threshold * 2
    scale = screen_width / world_width
    return int(env.state[0] * scale + screen_width / 2.0)  # MIDDLE OF CART

if __name__ == "__main__":
    total_steps = 10000
    e_greedy_parameters = EpsilonGreedyParameters(.9, 0.05, 200)
    discount = 0.999
    #lr = 0.01
    buffer_size = 10000
    replay_per_step = 128
    image_size = (40, 90)
    window_size = (600, 400)
    episodes_per_copy = 10

    display_game = False

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    transforms = tr.Compose([
        tr.ToPILImage(),
        tr.Grayscale(),
        tr.Resize(40, Image.CUBIC),
        tr.ToTensor()
    ])

    env = gym.make('CartPole-v1')

    action_set = set([x for x in range(env.action_space.n)])

    policy_agent = MyLovelyAgent(image_size, action_set,
                                 e_greedy_parameters, device)
    target_agent = policy_agent.copy()
    target_agent.eval()

    optimizer = torch.optim.RMSprop(policy_agent.parameters())
    buffer = ExperienceBuffer(buffer_size)
    teacher = MrAdamsTheTeacher(target_agent, policy_agent, discount,
                                optimizer, device, buffer, replay_per_step)

    accumulated_reward = 0
    inner_step = 0
    episodes = 0
    episode_reward_ma = []
    ma_records = []
    total_reward = []
    episode_loss = []
    max_ma_reward = 0
    ma_length = 100

    plt.show()
    plot(total_reward, ma_records)
    env.reset()
    proc_image = make_state_image(env, transforms)
    loop_start = datetime.now()
    for t in range(total_steps):
        inner_step += 1

        if display_game:
            env.render("human")

        action = policy_agent.make_up_my_mind(proc_image.unsqueeze(0))

        _, reward, done, info = env.step(action)
        accumulated_reward += reward
        new_proc_image = make_state_image(env, transforms)

        buffer.append(Experience(
            proc_image, new_proc_image, reward, action, done))
        proc_image = new_proc_image

        loss = teacher.teach_multiple()
        if loss:
            episode_loss.append(loss.item())

        if done:
            total_reward.append(accumulated_reward)
            episode_reward_ma.append(accumulated_reward)
            if len(episode_reward_ma) > ma_length:
                episode_reward_ma.pop(0)
            ma = sum(episode_reward_ma)/len(episode_reward_ma)
            ma_records.append(ma)
            if ma > max_ma_reward:
                max_ma_reward = ma

            if episodes % episodes_per_copy == 0:
                print("Copying value function")
                teacher.target.load_state_dict(policy_agent.state_dict())

            episodes += 1
            avg_loss = sum(episode_loss) / \
                len(episode_loss) if episode_loss else 0
            print(
                f"{episodes} - {t+1} - Steps Took: {inner_step} | AccumR: {accumulated_reward} | MA{ma_length}: {ma:.2f} | Loss: {avg_loss}")
            plot(total_reward, ma_records)
            env.reset()
            accumulated_reward = 0.
            inner_step = 0
            e_greedy_parameters.step()
            episode_loss.clear()

    env.close()
    loop_end = datetime.now()

    elapsed_time = (loop_end - loop_start).total_seconds()
    print(
        f"Elapsed Time: {elapsed_time} | Max Reward MA{ma_length}: {max_ma_reward}")

    plt.plot(total_reward, label="Reward Per Episode")
    plt.plot(ma_records, label="MA 100")
    plt.legend()
    plt.show()


import tweepy
from textblob import TextBlob
import preprocessor as p
import statistics
from typing import List

from keys import consumer_key, consumer_secret

print(consumer_key)
print(consumer_secret)

auth = tweepy.AppAuthHandler(consumer_key, consumer_secret)
api = tweepy.API(auth)

def get_tweets(keyword: str) -> List[str]:

    all_tweets = []

    for tweet in tweepy.Cursor(api.search_tweets, q=keyword, tweet_mode='extended', lang='es').items(100):
        all_tweets.append(tweet.full_text)

   # for tweet in all_tweets:
    #    print(tweet)

    return all_tweets

def clean_tweets(all_tweets: List[str]) -> List[str]:

    tweets_clean = []
    for tweet in all_tweets:
        tweets_clean.append(p.clean(tweet))

    return tweets_clean

def get_sentiment(all_tweets: List[str]) -> List[float]:

    sentiment_scores = []

    for tweet in all_tweets:
        blob = TextBlob(tweet)
        sentiment_scores.append(blob.sentiment.polarity)

    return sentiment_scores

def generate_average_sentiment_score(keyword: str) -> int:

    tweets = get_tweets(keyword)
    tweets_clean = clean_tweets(tweets)
    sentiment_scores = get_sentiment(tweets_clean)

    average_score = statistics.mean(sentiment_scores)

    return average_score

if __name__ == "__main__":

    print("Primera Keyword:")
    first_thing = input()
    print("Segunda Keyword:")
    second_thing = input()
    print("\n")

    first_score = generate_average_sentiment_score(first_thing)
    second_score = generate_average_sentiment_score(second_thing)

    print(f"Puntuacion Primera Keyword: {first_score}")
    print(f"Puntuacion Segunda Keyword: {second_score}")


import mimetypes
import re
from threading import Thread

from PySide6.QtCore import Qt, Slot
from PySide6.QtWidgets import (QDialog, QFileDialog, QGridLayout, QLabel,
                               QLineEdit, QMainWindow, QMessageBox,
                               QPushButton, QStackedWidget, QVBoxLayout,
                               QWidget)

from audio_cutter import AudioCutter
from cut_files_view import CutFilesView
from select_files_view import SelectFilesView

class MainWindow(QMainWindow):
    def __init__(self) -> None:
        super().__init__()
        self.audio_cutter = AudioCutter()

        self.create_layout()

        self.select_files_view.files_selected.connect(self.on_files_selected)

    def create_layout(self) -> None:
        self.setWindowTitle("App")
        self.resize(600, 200)

        self.stack = QStackedWidget()
        self.select_files_view = SelectFilesView(self.audio_cutter)
        self.cut_files_view = CutFilesView(self.audio_cutter)

        self.stack.addWidget(self.select_files_view)
        self.stack.addWidget(self.cut_files_view)
        self.setCentralWidget(self.stack)

        self.stack.setCurrentWidget(self.select_files_view)

    @Slot()
    def on_files_selected(self) -> None:
        self.stack.setCurrentWidget(self.cut_files_view)
        # Running in a separate thread so as not to block UI.
        # thread = Thread(target=self.audio_cutter.start_cutting)
        # thread.start()
        self.audio_cutter.start_cutting()


# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Private helpers for futures."""

import logging
import threading

_LOGGER = logging.getLogger(__name__)

def start_daemon_thread(*args, **kwargs):
    """Starts a thread and marks it as a daemon thread."""
    thread = threading.Thread(*args, **kwargs)
    thread.daemon = True
    thread.start()
    return thread

def safe_invoke_callback(callback, *args, **kwargs):
    """Invoke a callback, swallowing and logging any exceptions."""
    # pylint: disable=bare-except
    # We intentionally want to swallow all exceptions.
    try:
        return callback(*args, **kwargs)
    except Exception:
        _LOGGER.exception('Error while executing Future callback.')


import time
import sys
import os
import datetime
from watchdog.observers import Observer
from watchdog.events import PatternMatchingEventHandler

def get_work_dir(folder):
    """
    Gets working directory is correct format depending on used operating system.
    :param folder: Folder name to get full path to (e.g Desktop, Downloads)
    :return: str: Full path to specified folder
    """
    work_dir = None
    if os.name == 'posix':
        work_dir = os.path.join(os.path.join(os.path.expanduser('~')), folder)
    elif os.name == 'nt':
        work_dir = os.path.join(os.path.join(os.environ['USERPROFILE']), folder)
    else:
        print('App will not run on this OS. Sorry. Exiting')
    return work_dir

def get_file_attr(f_src):
    """
    Gets files attributes (name, extension (type), source path) from full path.
    :param f_src: Full path to file
    :return: list, str. File name and extension; Full path to file
    """
    return f_src.split("\\")[-1].split(".")[-2:], f_src

def move_to_folder(f_name_ext, f_src, folder):
    """
    Main task: moves new file from observed path to specified folder in Documents
    Following sequence:
    1. Checks if specified folder is created in Documents. If not create it.
    2. Checks if file with the same name exists in specified folder. If exists add
    full today's date to new file (e.g file_2019_08_22_12_42_11).
    3. Move file to specified folder.
    :param f_name_ext: list containing file name and file type
    :param f_src: str full path to file
    :param folder: str folder name to move file to
    :return: nothing
    """

    f_current_src = f_src
    f_path = f'{get_work_dir("Documents")}/{folder}'
    if not os.path.exists(f_path):
        os.mkdir(f_path)
    if os.path.exists(f'{f_path}/{".".join(f_name_ext)}'):
        f_new_src = f'{f_path}/{f_name_ext[0]}_{datetime.datetime.now().strftime("%Y_%m_%d_%H_%m_%S")}.{f_name_ext[1]}'
    else:
        f_new_src = f'{f_path}/{".".join(f_name_ext)}'
    print(f'Moving file {".".join(f_name_ext)} to {folder}')
    os.rename(f_current_src, f_new_src)

class DesktopHandler(PatternMatchingEventHandler):
    """
    DesktopHandler inherits from on of the existing handlers - PatterMatchingEventHandler.
    """

    def process(self, event):
        """
        Process incoming events occurring in observing directory. If event is directory created
        processing is skipped. If even is a file created then file is moved to specified folder by type.
        Available types:
        images (jpg, png, gif, ..), documents (doc, docx, pdf, odt), video (mp4, avi, ..)
        sound (mp3, flac, ..), other if not falls into any categories
        :param event: object with 3 attributes (type, is_directory, src_path)
        :return: nothing
        """
        if event.is_directory:
            print('Directory is created. Skipping processing!')
            return
        f_name_ext, f_src = get_file_attr(event.src_path)
        dic_of_type = {'IMAGES': ['jpg', 'png', 'gif'],
                       'DOCUMENTS': ['doc', 'docx', 'pdf', 'odt', 'txt'],
                       'VIDEO': ['mp4', 'mov', 'avi'],
                       'SOUND': ['mp3', 'flac']}

        for folder, list_of_formats in dic_of_type.items():
            if f_name_ext[1] in list_of_formats:
                print(f'Moving file {".".join(f_name_ext)} to {folder}')
                time.sleep(1)
                return move_to_folder(f_name_ext, f_src, folder)
        return move_to_folder(f_name_ext, f_src, 'OTHER')

    def on_created(self, event):
        """
        Method is executed when a file or directory is created. Then method
        calls process method which process occurring event.
        :param event: object indicating event
        :return: nothing
        """
        self.process(event)

if __name__ == '__main__':
    print('----------------------------------------------------------------------')
    print('You can specify directory to which to clean by providing an argument')
    print('e.g python main.py /home/my_secret_folder')
    print('Otherwise it will use Desktop directory for Windows and Linux')
    print('----------------------------------------------------------------------')
    args = sys.argv[1:]
    watch_dir = None
    if args:
        watch_dir = args[0]
    else:
        watch_dir = get_work_dir('Desktop')
    print(f'Observing directory: {watch_dir}')
    """
    Observer class will watch for any change (in our case on_created), and then dispatch
    the event to specified handled. In our case - DesktopHandler()
    """
    observer = Observer()
    observer.schedule(DesktopHandler(), path=watch_dir if watch_dir else sys.exit(1))
    observer.start()

    try:
        while True:
            time.sleep(1)
    except KeyboardInterrupt:
        observer.stop()
        observer.join()


import torchvision
import torchvision.transforms as transforms
import pandas as pd
from utils import IMAGE_PREPROCESSING
from torch.utils.data import DataLoader
import numpy as np

SCENARIO_1 = [0.1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
SCENARIO_2 = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.1]
SCENARIO_3 = [0.1, 0.01, 0.3, 0.4, 0.5, 0.6, 0.1, 0.8, 0.9, 1]

SAVE_PATH = "scenarios"

def download_cifar_set(
        transform_img: transforms.Compose, save: bool, savepath: str, is_train: bool, batch: int) -> DataLoader:
    trainset = torchvision.datasets.CIFAR10(
        root=savepath, train=is_train, download=save, transform=transform_img)
    return DataLoader(
        trainset, batch_size=batch, shuffle=is_train, num_workers=2)

def reduce_dataset_over_strategy(
        data_loader: DataLoader,
        strategy: list,
        filename: str
):
    """Change dataset contents according to strategy and save it to numpy file.

    :param data_loader: data loader of dataset in which should be data and targets changed.
    :type data_loader: DataLoader
    :param strategy: list containing ratio of data should remain in given class.
    :param filename: path where the files are stored with proper scenario name.
    :type filename: str
    :return: returns targets and images lists that were made smaller depending on strategy.
    :rtype: tuple(list, list[ndarray])

    """
    data_ext = [

    ]
    for _, targets in data_loader:
        targets_list = targets.numpy()
        data_ext.extend(targets_list)

    df = pd.DataFrame(data_ext, columns=['original_label'])
    classes_indexes = {k: df.index[df['original_label'] == k].to_list()
                       for k in range(0, len(strategy)) if 1 - strategy[k] != 0}

    images = data_loader.dataset.data
    targets = np.array(data_loader.dataset.targets)

    indexes_to_remove = []
    for k, indexes in classes_indexes.items():
        step = int(100 / (100 * strategy[k]))
        # print(step)
        indexes_to_remove += indexes[::step]
    print(len(indexes_to_remove))
    images = np.delete(images, indexes_to_remove, axis=0)
    targets = np.delete(targets, indexes_to_remove, axis=0)

    from collections import defaultdict
    class_count = defaultdict(int)

    for target in targets:
        class_count[target] += 1
    print(class_count)

    np.save(
        f"{SAVE_PATH}/{filename}-images", images
    )
    np.save(
        f"./{SAVE_PATH}/{filename}-targets", targets
    )
    return targets, images

def load_subset_dataset(data_loader: DataLoader, path: str):
    """ Load chosen scenario from numpy files and store them in dataset.

        :param data_loader: data loader of dataset in which should be data and targets changed.
        :type data_loader: DataLoader
        :param path: path where the files are stored with proper scenario name.
        :type path: str
        :return: None, it only changes contents of dataloader

    """
    images = np.load(f"{path}-images.npy")
    targets = np.load(f"{path}-targets.npy")
    data_loader.dataset.targets = targets

    unique_values, counts = np.unique(targets, return_counts=True)
    for i in range (0, 9):
        print("{}: {}".format(unique_values[i], counts[i]))
    data_loader.dataset.data = images

def generate_imbalance(dataset, strategy: list, per_class_indices: int, filename: str | None = None):

    targets = np.array(dataset.targets)
    classes, class_counts = np.unique(targets, return_counts=True)
    nb_classes = len(classes)

    # Create artificial imbalanced class counts
    # imbal_class_counts = [per_class_indices] * 10

    imbal_class_counts = [int(rate * per_class_indices) for rate in strategy]

    print(imbal_class_counts)

    # Get class indices
    class_indices = [np.where(targets == i)[0] for i in range(nb_classes)]
    # Get imbalanced number of instances
    imbal_class_indices = []
    for class_idx, class_count in zip(class_indices, imbal_class_counts):
        imbal_class_indices.extend(class_idx[:class_count])

    # Set target and data to dataset
    dataset.targets = targets[imbal_class_indices]
    dataset.data = dataset.data[imbal_class_indices]
    assert len(dataset.targets) == len(dataset.data)
    if filename is not None:
        np.save(
            f"{SAVE_PATH}/{filename}-images", dataset.data
        )
        np.save(
            f"./{SAVE_PATH}/{filename}-targets", targets[imbal_class_indices]
        )
    return dataset

if __name__ == "__main__":

    arr= np.load("./out/strategy_many_classes-targets.npy")
    unique_values, counts = np.unique(arr, return_counts=True)
    print(counts)
    dataset = download_cifar_set(IMAGE_PREPROCESSING, True, "./data", True, 50000).dataset
    generate_imbalance(dataset, SCENARIO_1, 5000, "strategy_one_class")
    generate_imbalance(dataset, SCENARIO_3, 5000, "strategy_three_class")
    generate_imbalance(dataset, SCENARIO_2, 5000, "strategy_many_classes")
    # reduce_dataset_over_strategy(dataset_2, SCENARIO_1, "strategy_one_class")
    # reduce_dataset_over_strategy(dataset_2, SCENARIO_3, "strategy_two_class")
    # reduce_dataset_over_strategy(dataset_2, SCENARIO_2, "strategy_many_classes")


from google.oauth2 import service_account
from google.auth.transport.requests import AuthorizedSession
db = "https://embedded-systems-cf93d-default-rtdb.europe-west1.firebasedatabase.app/"
keyfile = "embedded-systems-cf93d-firebase-adminsdk-amky8-e0a50b80ba.json"
scopes = [
    "https://www.googleapis.com/auth/userinfo.email",
    "https://www.googleapis.com/auth/firebase.database"
]

credentials = service_account.Credentials.from_service_account_file(keyfile, scopes=scopes)

authed_session = AuthorizedSession(credentials)

def updatefree(space, status):
    path = f"parkingspaces/{space}.json"

    if status == 'true':
        update = {"Free": "True"}
    else:
        update = {"Free": "False"}

    resp = authed_session.patch(db + path, json=update)

    if resp.ok:
        print("Updated free status of space {}".format(space))
    else:
        raise


import os
import numpy as np

import torch
import torchvision.datasets as dset
import torchvision.transforms as transforms

#from torchsample.datasets import TensorDataset

from utils import Invert
from utils import Gray

DATA_PATH = 'data'
def data_path(folder):
    return os.path.join(DATA_PATH, folder)

class PatchDataset:

    def __init__(self, dataset, patch_size):
        self.dataset = dataset
        self.patch_size = patch_size
        self.nc, self.h, self.w = dataset[0][0].size()
        self.nb = len(dataset)

    def __getitem__(self, i):
        im, y = self.dataset[i]
        #y = np.random.randint(0, self.h // self.patch_size) * self.patch_size
        #x = np.random.randint(0, self.w // self.patch_size) * self.patch_size
        y = np.random.randint(0, self.h - self.patch_size + 1)
        x = np.random.randint(0, self.w - self.patch_size + 1)
        patch = im[:, x:x + self.patch_size, y:y + self.patch_size]
        return patch, y

    def __len__(self):
        return len(self.dataset)

class H5Dataset:

    def __init__(self, X, y, transform=lambda x:x):
        self.X = X
        self.y = y
        self.transform = transform

    def __getitem__(self, index):
        return self.transform(torch.from_numpy(self.X[index])), self.y[index]

    def __len__(self):
        return len(self.X)

def load_dataset(dataset_name, split='full'):
    if dataset_name == 'mnist':
        dataset = dset.MNIST(
            root=data_path('mnist'),
            download=True,
            transform=transforms.Compose([
                transforms.Scale(32),
                transforms.ToTensor(),
            ])
        )
        return dataset
    elif dataset_name == 'celeba_h5':
        dataset = _load_h5('data/celeba64_align.h5')
        return dataset
    elif dataset_name == 'coco':
        dataset = dset.ImageFolder(root=data_path('coco'),
            transform=transforms.Compose([
            transforms.Scale(64),
            transforms.CenterCrop(64),
            transforms.ToTensor(),
            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
         ]))
        return dataset
    elif dataset_name == 'coco_256':
        dataset = dset.ImageFolder(root=data_path('coco'),
            transform=transforms.Compose([
            transforms.Scale(256),
            transforms.CenterCrop(256),
            transforms.ToTensor(),
            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
         ]))
        return dataset
    elif dataset_name == 'footwear':
        dataset = dset.ImageFolder(root=data_path('shoes/ut-zap50k-images'),
            transform=transforms.Compose([
            transforms.Scale(64),
            transforms.CenterCrop(64),
            transforms.ToTensor(),
            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
         ]))
        return dataset
    elif dataset_name == 'footwear_32':
        dataset = dset.ImageFolder(root=data_path('shoes/ut-zap50k-images'),
            transform=transforms.Compose([
            transforms.Scale(32),
            transforms.CenterCrop(32),
            transforms.ToTensor(),
            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
         ]))
        return dataset
    elif dataset_name == 'celeba':
        dataset = dset.ImageFolder(root=data_path('celeba'),
            transform=transforms.Compose([
            transforms.Scale(78),
            transforms.CenterCrop(64),
            transforms.ToTensor(),
            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
         ]))
        return dataset
    elif dataset_name == 'celeba_128':
        dataset = dset.ImageFolder(root=data_path('celeba'),
            transform=transforms.Compose([
            transforms.Scale(128),
            transforms.CenterCrop(128),
            transforms.ToTensor(),
            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
         ]))
        return dataset
    elif dataset_name == 'celeba_256':
        dataset = dset.ImageFolder(root=data_path('celeba'),
            transform=transforms.Compose([
            transforms.Scale(256),
            transforms.CenterCrop(256),
            transforms.ToTensor(),
            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
         ]))
        return dataset

    elif dataset_name == 'birds':
        dataset = dset.ImageFolder(root=data_path('birdsfull'),
            transform=transforms.Compose([
            transforms.Scale(64),
            transforms.CenterCrop(64),
            transforms.ToTensor(),
            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
         ]))
        return dataset
    elif dataset_name == 'fonts':
        dataset = dset.ImageFolder(root=data_path('fonts/full'),
            transform=transforms.Compose([
            transforms.ToTensor(),
            Invert(),
            Gray(),
         ]))
        return dataset
    else:
        dataset = dset.ImageFolder(root=data_path(dataset_name),
            transform=transforms.Compose([
            transforms.Scale(256),
            transforms.CenterCrop(256),
            transforms.ToTensor(),
        ]))
        return dataset
def _load_npy(filename):
    data = np.load(filename)
    X = torch.from_numpy(data['X']).float()
    if 'y' in data:
        y  = torch.from_numpy(data['y'])
    else:
        y = torch.zeros(len(X))
    X /= X.max()
    X = X * 2 - 1
    print(X.min(), X.max())
    dataset = dset.TensorDataset(
        inputs=X,
        targets=y,
    )
    return dataset

def _load_h5(filename):
    import h5py
    data = h5py.File(filename, 'r')
    X = data['X']
    if 'y' in data:
        y  = (data['y'])
    else:
        y = np.zeros(len(X))
    dataset = H5Dataset(X, y, transform=lambda u:2*(u.float()/255.)-1)
    return dataset


from freezegun import freeze_time

from database import orders
from database.schemas import Chips
from database.schemas import Drink
from database.schemas import ExtraCheese
from database.schemas import ExtraTomato
from database.schemas import Hamburger
from database.schemas import Meat
from database.schemas import Order
from tests.utils import assert_dicts

@freeze_time('2020-02-10')
def test_calculate_prices_1(order_1):
    expected = Order(
        barista='txema',
        hamburgers=[
            Hamburger(
                meats=[
                    Meat(
                        type='pollo',
                        size='250g',
                        cooked='al punto',
                        price=2.5,
                    )
                ],
                extra_cheese=ExtraCheese(
                    type='cheddar',
                    price=1.5,
                ),
                extra_tomato=ExtraTomato(
                    type='normal',
                    price=1.0,
                ),
                price=10.0,
            ),
        ],
        chips=[
            Chips(
                type='gajo',
                size='pequeñas',
                price=2.5,
            ),
        ],
        price=12.5,
        promotions=[],
    )
    result = orders.calculate_prices(order=order_1)
    assert_dicts(original=result.dict(), expected=expected.dict())

@freeze_time('2020-02-10')
def test_calculate_prices_2(order_2):
    expected = Order(
        barista='txema',
        hamburgers=[
            Hamburger(
                meats=[
                    Meat(
                        type='cerdo',
                        size='250g',
                        cooked='al punto',
                        price=2.5,
                    ),
                    Meat(
                        type='cerdo',
                        size='380g',
                        cooked='al punto',
                        price=3.5,
                    ),
                ],
                extra_tomato=ExtraTomato(
                    type='normal',
                    price=1.0,
                ),
                price=12.0 * (100 - 15) / 100,
            ),
        ],
        chips=[
            Chips(
                type='de la abuela',
                size='grandes',
                price=3.5,
            ),
        ],
        drinks=[
            Drink(
                type='burribeer',
                price=2.5,
            ),
        ],
        price=16.2,
        promotions=['burrimenu'],
    )
    result = orders.calculate_prices(order=order_2)
    assert_dicts(original=result.dict(), expected=expected.dict())

@freeze_time('2020-02-10')
def test_calculate_prices_3(order_3):
    expected = Order(
        barista='txema',
        chips=[
            Chips(
                type='deluxe',
                size='grandes',
                price=3.5,
            ),
            Chips(
                type='gajo',
                size='grandes',
                price=3.5,
            ),
        ],
        drinks=[
            Drink(
                type='burribeer',
                price=2.5,
            ),
        ],
        price=9.5,
        promotions=[],
    )
    result = orders.calculate_prices(order=order_3)
    assert_dicts(original=result.dict(), expected=expected.dict())

@freeze_time('2020-10-01 17:00:00')
def test_order_is_ready_listo(order_data):
    assert orders.order_is_ready(order=order_data) == 'listo'


from typing import Any, List, Tuple

from auth.datadef import UserInfo
from base.model import db
from base.store import Store
from connector.gino import get_bind
from ..cfg import logger
from ..command import Command
from ..context import Context
from ..datadef import Initiator
from ..entity import Entity
from ..event import Event, EventEntity
from ..proxy import Proxy
from ..response import BaseResponse, Response
from ..statemgr.state_manager import StateMgr
from .domain_handler import DomainHandler
from .domain_register import DomainRegister

class DomainProcess(DomainRegister, DomainHandler):
    def __init__(self, user: UserInfo, initiator: Initiator):
        self.__store__ = Store()
        self.__context__ = Context(self, user, initiator)
        self.__response__ = Response()
        self.__statemgr__ = StateMgr(self)
        self.__proxy__ = Proxy(self, self.__statemgr__)
        self.__command__ = Command(self.__context__, self.__proxy__)
        self.__event__ = Event(self, self.__statemgr__)

    def process_command(self, entity_name: str, data: dict):
        logger.debug("> Process command")
        Entity = self.lookup_entity(entity_name)
        entity = Entity(data=data)
        command = self.lookup_command_handler(Entity)
        self.command.push((self.__namespace__, command, entity))
        # command = self.lookup_command_handler(entity)
        # [command_process for command_process in command.commit()]

    async def commit(self) -> List[BaseResponse]:
        async def distribute_aggregate(aggregate: Tuple[str, Entity, Any]):
            if aggregate is None:
                return
            logger.debug(
                "~~~> Aggregate: %r"
                % (
                    (
                        aggregate[0],
                        aggregate[1].__class__.__name__,
                        aggregate[2].__name__,
                    ),
                )
            )
            if isinstance(aggregate[1], EventEntity):
                await self.event.execute(aggregate)
            if isinstance(aggregate[1], BaseResponse):
                self.response.push(aggregate)

        async def _commit():
            async for aggregate in self.command.commit():
                await distribute_aggregate(aggregate)
            # await self.event.commit()
            state_msg = [result async for result in self.statemgr.commit()]
            logger.info(">>>> State_msg: \n%r", state_msg)
            return [resp async for resp in self.response.commit()]

        con = await get_bind()
        async with con.transaction():
            return await _commit()

    @property
    def command(self) -> Command:
        return self.__command__

    @property
    def event(self) -> Event:
        return self.__event__

    @property
    def store(self) -> Store:
        return self.__store__

    @property
    def context(self) -> Context:
        return self.__context__

    @property
    def response(self) -> Response:
        return self.__response__

    @property
    def proxy(self) -> Proxy:
        return self.__proxy__


# use yahoo_fin library to parse data

import yahoo_fin.stock_info as si
from datetime import datetime, timedelta

def convert_currency(src, dst, amount):
    # construct the currency pair symbol
    symbol = f'{src} {dst}=X'

    # use get_data() from stock_info to pass the symbol of currency
    # pass "1m" as the interval parameter in the get_data() method to retrieve data for the current minute
    # not the default current day
    latest_data = si.get_data(symbol, interval='1m', start_date=datetime.now() - timedelta(days=2))
    # extract minute data from the last 2 days, because there be problems with the course on the weekend

    # get the latest datetime
    latest_datetime = latest_data.index[-1].to_pydatetime()

    # get the last price
    last_price = latest_data.iloc[-1].close

    # return the latest data and converted amount
    return latest_datetime, last_price * amount

if __name__ == "__main__":
    import sys
    # get currency and amount
    source_currency = sys.argv[1]
    destination_currency = sys.argv[2]

    amount = float(sys.argv[3])

    latest_datetime, exchange_rate = convert_currency(source_currency, destination_currency, amount)

    print(f'Last update time: {latest_datetime}')
    print(f'{amount} {source_currency} = {exchange_rate} {destination_currency}')


from flask_restful import Resource

from flask_restful import request, abort
from datetime import datetime

# models
from apps.api.models import db
from apps.api.models.models import App

# resources
from apps.api.endpoints.app.fields import all_apps, marshal_with
from apps.api.endpoints.app.parsers import post_app, put_app

# security
from apps.api.endpoints.security import decorators as auth

class AppEndpoint(Resource):

    @auth.login_required
    @marshal_with(all_apps)
    def get(self):
        return db.session.query(App).limit(25).all()

    @auth.login_required
    @marshal_with(all_apps)
    def post(self):
        args = post_app()
        try:
            posted = App()
            posted.name = args.name
            posted.public_key = args.public_key
            if 'is_active' in request.json:
                posted.is_active = args.is_active
            else:
                posted.is_active = False
            posted.created_at = datetime.now()
            db.session.add(posted)
            db.session.commit()
            return db.session.query(App).limit(25).all()
        except Exception as e:
            return abort(400, message='Your request has been failed')

class AppUidEndpoint(Resource):

    @auth.login_required
    @marshal_with(all_apps)
    def get(self, id):
        return db.session.query(App).get(id)

    @auth.login_required
    @marshal_with(all_apps)
    def delete(self, id):
        delete_app = db.session.query(App).get(id)
        db.session.delete(delete_app)
        db.session.commit()
        return db.session.query(App).limit(25).all()

    @auth.login_required
    @marshal_with(all_apps)
    def put(self, id):
        try:
            args = put_app()
            update_app = db.session.query(App).get(id)
            update_app.name = args.name
            update_app.public_key = args.public_key
            if 'is_active' in request.json:
                update_app.is_active = args.is_active
            else:
                update_app.is_active = False
            db.session.commit()
            return db.session.query(App).limit(25).all()
        except Exception as e:
            return abort(400, message='Your request has been failed')


import pymongo, datetime, certifi, requests
from bs4 import BeautifulSoup
from variables import mongo_string
from data import stocks_list

# stocks_list = ['ANWARGALV', 'BSRMSTEEL']

myclient = pymongo.MongoClient(mongo_string, tlsCAFile=certifi.where())
mydb = myclient["stockanalyst"]

data_setting = mydb.settings.find_one()

if data_setting['dataInsertionEnable'] == 0:
  print('exiting script')
  exit()

for stock in stocks_list:
  stock_data = mydb.fundamentals.find_one({
    'tradingCode': stock
  })

  if 'epsQuaterly' in stock_data :
    eps_data = sorted(stock_data['epsQuaterly'], key=lambda d: str(d['year']), reverse=True)[0]

    count = 0

    if 'q4' in eps_data:
      count = count + 1
    if 'q3' in eps_data:
      count = count + 1
    if 'q2' in eps_data:
      count = count + 1
    if 'q1' in eps_data:
      count = count + 1

    if count == 0:
      eps = 0
    else:
      total = eps_data['q1'] if 'q1' in eps_data else 0 + eps_data['q2'] if 'q2' in eps_data else 0 + eps_data['q3'] if 'q3' in eps_data else 0 + eps_data['q4'] if 'q4' in eps_data else 0

      eps = round((total / count) * 4, 3)

    # print(stock, eps)

    myquery = { 'tradingCode': stock }

    newvalues = { "$set": {
        "epsCurrent": eps,
    } }

    mydb.fundamentals.update_one(myquery, newvalues)

  else:
    print(stock, 'error occured!')


import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import argparse
import torch.utils.data.sampler as sampler

from create_dataset import *
from utils import *

parser = argparse.ArgumentParser(description='Multi-task: Attention Network')
parser.add_argument('--weight', default='equal', type=str, help='multi-task weighting: equal, uncert, dwa, pcgrad')
parser.add_argument('--dataroot', default='/data/dataset/nyuv2', type=str, help='dataset root')
parser.add_argument('--temp', default=2.0, type=float, help='temperature for DWA (must be positive)')
parser.add_argument('--apply_augmentation', action='store_true', help='toggle to apply data augmentation on NYUv2')
parser.add_argument('--apply_weightdeacay', action='store_true', help='toggle to apply weight decay 1e-5 on learning rate')
parser.add_argument('--apply_pcgrad', action='store_true', help='toggle to apply gradient surgery')

opt = parser.parse_args()

import torch
import resnet

from resnet_dilated import ResnetDilated
#from aspp import DeepLabHead
from resnet import Bottleneck, conv1x1

class MTANDeepLabv3(nn.Module):
    def __init__(self):
        super(MTANDeepLabv3, self).__init__()
        backbone = ResnetDilated(resnet.__dict__['resnet18'](pretrained=True))
        ch = [64,128,256,512]
        #ch = [256, 512, 1024, 2048]

        self.class_nb = 13
        self.logsigma = nn.Parameter(torch.FloatTensor([-0.5, -0.5, -0.5]))

        self.tasks = ['1', '2', '3', '4', '5','6','7','8','9','10']
        #self.num_out_channels = {'segmentation': 13, 'depth': 1, 'normal': 3}

        self.shared_conv = nn.Sequential(backbone.conv1, backbone.bn1, backbone.relu1, backbone.maxpool)

        # We will apply the attention over the last bottleneck layer in the ResNet.
        self.shared_layer1_b = backbone.layer1[:-1]
        self.shared_layer1_t = backbone.layer1[-1]

        self.shared_layer2_b = backbone.layer2[:-1]
        self.shared_layer2_t = backbone.layer2[-1]

        self.shared_layer3_b = backbone.layer3[:-1]
        self.shared_layer3_t = backbone.layer3[-1]

        self.shared_layer4_b = backbone.layer4[:-1]
        self.shared_layer4_t = backbone.layer4[-1]

        # Define task specific attention modules using a similar bottleneck design in residual block
        # (to avoid large computations)
        self.encoder_att_1 = nn.ModuleList([self.att_layer(ch[0], ch[0] // 4, ch[0]) for _ in self.tasks])
        self.encoder_att_2 = nn.ModuleList([self.att_layer(2 * ch[1], ch[1] // 4, ch[1]) for _ in self.tasks])
        self.encoder_att_3 = nn.ModuleList([self.att_layer(2 * ch[2], ch[2] // 4, ch[2]) for _ in self.tasks])
        self.encoder_att_4 = nn.ModuleList([self.att_layer(2 * ch[3], ch[3] // 4, ch[3]) for _ in self.tasks])

        # Define task shared attention encoders using residual bottleneck layers
        # We do not apply shared attention encoders at the last layer,
        # so the attended features will be directly fed into the task-specific decoders.
        self.encoder_block_att_1 = self.conv_layer(ch[0], ch[1] // 4)
        self.encoder_block_att_2 = self.conv_layer(ch[1], ch[2] // 4)
        self.encoder_block_att_3 = self.conv_layer(ch[2], ch[3] // 4)

        self.down_sampling = nn.MaxPool2d(kernel_size=2, stride=2)
        self.final_pool = nn.AdaptiveAvgPool2d((1, 1))
        self.classfier = nn.ModuleList([nn.Linear(512, 10) for _ in self.tasks])
         # or 20

        # Define task-specific decoders using ASPP modules
        #self.decoders = nn.ModuleList([DeepLabHead(2048, self.num_out_channels[t]) for t in self.tasks])

    # def forward(self, x, out_size):
    def forward(self, x, task_id):
        img_size  = x.size()[-2:]
        # Shared convolution
        x = self.shared_conv(x)

        # Shared ResNet block 1
        u_1_b = self.shared_layer1_b(x)
        u_1_t = self.shared_layer1_t(u_1_b)

        # Shared ResNet block 2
        u_2_b = self.shared_layer2_b(u_1_t)
        u_2_t = self.shared_layer2_t(u_2_b)

        # Shared ResNet block 3
        u_3_b = self.shared_layer3_b(u_2_t)
        u_3_t = self.shared_layer3_t(u_3_b)

        # Shared ResNet block 4
        u_4_b = self.shared_layer4_b(u_3_t)
        u_4_t = self.shared_layer4_t(u_4_b)

        # Attention block 1 -> Apply attention over last residual block
        a_1_mask = [att_i(u_1_b) for att_i in self.encoder_att_1]  # Generate task specific attention map
        a_1 = [a_1_mask_i * u_1_t for a_1_mask_i in a_1_mask]  # Apply task specific attention map to shared features
        a_1 = [self.down_sampling(self.encoder_block_att_1(a_1_i)) for a_1_i in a_1]

        # Attention block 2 -> Apply attention over last residual block
        a_2_mask = [att_i(torch.cat((u_2_b, a_1_i), dim=1)) for a_1_i, att_i in zip(a_1, self.encoder_att_2)]
        a_2 = [a_2_mask_i * u_2_t for a_2_mask_i in a_2_mask]
        a_2 = [self.encoder_block_att_2(a_2_i) for a_2_i in a_2]

        # Attention block 3 -> Apply attention over last residual block
        a_3_mask = [att_i(torch.cat((u_3_b, a_2_i), dim=1)) for a_2_i, att_i in zip(a_2, self.encoder_att_3)]
        a_3 = [a_3_mask_i * u_3_t for a_3_mask_i in a_3_mask]
        a_3 = [self.encoder_block_att_3(a_3_i) for a_3_i in a_3]

        # Attention block 4 -> Apply attention over last residual block (without final encoder)
        a_4_mask = [att_i(torch.cat((u_4_b, a_3_i), dim=1)) for a_3_i, att_i in zip(a_3, self.encoder_att_4)]
        a_4 = [a_4_mask_i * u_4_t for a_4_mask_i in a_4_mask]

        for i, t in enumerate(self.tasks):
            if t == task_id:
                z = torch.flatten(a_4[i], 1)
                out = self.classfier[i](z)
                return out

    def att_layer(self, in_channel, intermediate_channel, out_channel):
        return nn.Sequential(
            nn.Conv2d(in_channels=in_channel, out_channels=intermediate_channel, kernel_size=1, padding=0),
            nn.BatchNorm2d(intermediate_channel),
            nn.ReLU(inplace=True),
            nn.Conv2d(in_channels=intermediate_channel, out_channels=out_channel, kernel_size=1, padding=0),
            nn.BatchNorm2d(out_channel),
            nn.Sigmoid())

    def conv_layer(self, in_channel, out_channel):
        downsample = nn.Sequential(conv1x1(in_channel, 4 * out_channel, stride=1),
                                   nn.BatchNorm2d(4 * out_channel))
        return Bottleneck(in_channel, out_channel, downsample=downsample)

'''
        # Task specific decoders
        out = [0 for _ in self.tasks]
        for i, t in enumerate(self.tasks):
            # out[i] = F.interpolate(self.decoders[i](a_4[i]), size=out_size, mode='bilinear', align_corners=True)
            out[i] = F.interpolate(self.decoders[i](a_4[i]), img_size, mode='bilinear', align_corners=True)
            if t == 'segmentation':
                out[i] = F.log_softmax(out[i], dim=1)
            if t == 'normal':
                out[i] = out[i] / torch.norm(out[i], p=2, dim=1, keepdim=True)
        return [out[0], out[1], out[2]], self.logsigma
'''

# define model, optimiser and scheduler
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
Resnet_MTAN = MTANDeepLabv3().to(device)

from pcgrad import PCGrad

if opt.apply_pcgrad:
    optimizer = optim.Adam(Resnet_MTAN.parameters(), lr=1e-4)
    weight_optimizer = PCGrad(optimizer)
    print('Applying gradient surgery on optimizer.')
    scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=100, gamma=0.5)
else:
    if opt.apply_weightdeacay:
        optimizer = optim.Adam(Resnet_MTAN.parameters(), lr=1e-4, weight_decay=1e-5)
        print('Applying weight decay 1e-5 on learning rate.')
    else:
        optimizer = optim.Adam(Resnet_MTAN.parameters(), lr=1e-4)
        print('Not applying weight decay on learning rate.')
    scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=100, gamma=0.5)

print('Parameter Space: ABS: {:.1f}, REL: {:.4f}'.format(count_parameters(Resnet_MTAN),
                                                         count_parameters(Resnet_MTAN) / 24981069))
print('LOSS FORMAT: SEMANTIC_LOSS MEAN_IOU PIX_ACC | DEPTH_LOSS ABS_ERR REL_ERR | NORMAL_LOSS MEAN MED <11.25 <22.5 <30')

# define dataset

dataset_path = opt.dataroot
if opt.apply_augmentation:
    nyuv2_train_set = NYUv2(root=dataset_path, train=True, augmentation=True)
    print('Applying data augmentation on NYUv2.')
else:
    nyuv2_train_set = NYUv2(root=dataset_path, train=True)
    print('Standard training strategy without data augmentation.')

nyuv2_test_set = NYUv2(root=dataset_path, train=False)

batch_size = 8
nyuv2_train_loader = torch.utils.data.DataLoader(
    dataset=nyuv2_train_set,
    batch_size=batch_size,
    shuffle=True)

nyuv2_test_loader = torch.utils.data.DataLoader(
    dataset=nyuv2_test_set,
    batch_size=batch_size,
    shuffle=False)

# Train and evaluate multi-task network
if opt.apply_pcgrad:
    multi_task_trainer_pcgrad(nyuv2_train_loader,
                       nyuv2_test_loader,
                       Resnet_MTAN,
                       device,
                       weight_optimizer,
                       scheduler,
                       opt,
                       200)
else:
    multi_task_trainer(nyuv2_train_loader,
                       nyuv2_test_loader,
                       Resnet_MTAN,
                       device,
                       optimizer,
                       scheduler,
                       opt,
                       200)


from django.shortcuts import render
from django.http import HttpResponse

# Create your views here.
from chatterbot import ChatBot
from chatterbot.trainers import ListTrainer, ChatterBotCorpusTrainer

bot = ChatBot('chatbot', read_only=False,
              logic_adapters=[
                  {
                      'import_path':'chatterbot.logic.BestMatch'

                      }])

list_to_train = [
    "hi",
    "hi,there",
    "what is your name?",
    "i am jsut a chatbot"
]

chatterbotCorpusTrainer = ChatterBotCorpusTrainer(bot)

#list_trainer = ListTrainer(bot)
#list_trainer.train(list_to_train)
chatterbotCorpusTrainer .train('chatterbot.corpus.english')

def index(request):
    return render(request, 'blog/index.html')

def specific(request):
    return HttpResponse("list1")

def getResponse(request):
    userMessage = request.GET.get('userMessage')
    chatResponse = str(bot.get_response(userMessage))
    return HttpResponse(chatResponse)


import langchain
from aiogram import Bot, Dispatcher, executor, types

#Associar o bot ao código
bot = Bot(token = 'SEU_TOKEN')
dp = Dispatcher(bot)

#Lista de resposta
Mensagem =['Mensagem']

#Carregando os documentos utilizados para consulta pelo GPT
loader = langchain.document_loaders.DirectoryLoader('Directory/',
                                                    glob="**/*.txt",
                                                    loader_cls=langchain.document_loaders.TextLoader)
documents = loader.load()
len(documents)
text_splitter = langchain.text_splitter.RecursiveCharacterTextSplitter(chunk_size =2300,
                                                                       chunk_overlap  =10,
                                                                       length_function =len,
                                                                       separators="\n\n")
texts = text_splitter.split_documents(documents)

#Função para comparar a pergunta aos textos e trazer as respostas
def query_gpt(input):
    embeddings = langchain.embeddings.OpenAIEmbeddings(openai_api_key = '')
    doc_search = langchain.vectorstores.Chroma.from_documents(texts, embeddings)
    chain = langchain.VectorDBQA.from_chain_type(llm=langchain.OpenAI(openai_api_key = ''),
                                                 chain_type = 'stuff',
                                                 vectorstore = doc_search)
    Mensagem[0] = chain.run(input)

#Função para receber a pergunta do usuário e responde-la pelo telegram
@dp.message_handler()
async def gpt(message:types.message):
    query_gpt(message.text)
    await message.reply(Mensagem[0])

if __name__ =='__main__':
    executor.start_polling(dp)


import cv2,math,time,struct,argparse
from bitstream import BitStream
import numpy as np
from skimage.util.shape import view_as_blocks

parser = argparse.ArgumentParser(description='Converter')
parser.add_argument("file", help="Input video")
parser.add_argument("output", help="Optional output file", nargs='?', default="data.bin")
parser.add_argument("-s", "--show", help="Show dct results", action="store_true")
parser.add_argument("-p", "--play", help="Play video in dct", action="store_true")
parser.add_argument("-d", "--debug", help="Print dct results ", action="store_true")
parser.add_argument("-q", "--quality", help="Scale quantization matrix", dest="quality", type=float, default=1.0)
cmd_args = parser.parse_args()

infile = cmd_args.file
outfile = cmd_args.output

CLEAR = "\033[K"

print("Reading from %s"%infile)
vid = cv2.VideoCapture(infile)
fps = vid.get(cv2.CAP_PROP_FPS)
width = int(vid.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(vid.get(cv2.CAP_PROP_FRAME_HEIGHT))
nbframes = int(vid.get(cv2.CAP_PROP_FRAME_COUNT))
normal_size = nbframes*width*height
duration = nbframes/fps
print("Video: %i frames (%i:%i) @ %ffps (%ix%i)"%(nbframes,duration/60,duration%60,fps,width,height))
print("Writing to file: %s"%outfile)

def img2bin(image):
	gray = cv2.cvtColor(f, cv2.COLOR_BGR2GRAY)
	return (gray-128).astype(np.int8)

quant = np.array([[16,11,10,16,24,40,51,61],
				[12,12,14,19,26,58,60,55],
				[14,13,16,24,40,57,69,56],
				[14,17,22,29,51,87,80,62],
				[18,22,37,56,68,109,103,77],
				[24,36,55,64,81,104,113,92],
				[49,64,78,87,103,121,120,101],
				[72,92,95,98,112,100,103,99]])*cmd_args.quality

quant = np.rint(quant).astype(np.uint16)
print(quant)

# 3 -> Duplicate frame
# 2 -> Delta Frame
# 1 -> Huffman Frame
# 0 -> END

gbits = BitStream()
tempbs = BitStream()

symboltable = {}
valsymboltable = {}

frequency = {}
valfrequency = {}

rle_min_same = 16
dc_chunk_size = 8 # needs to divide width&height
dc_chunk_bits = int(math.log((width*height)//(dc_chunk_size*dc_chunk_size),2))
print("DC Chunk size: %i"%(dc_chunk_size))
print("DC Chunk bits: %i"%(dc_chunk_bits))

def addbits(bs,by,nb):
	for i in range(nb):
		bs.write((by >> (nb-1-i)) & 1 == 1)

def huffcomp(dcts,bs):
	bs.read()
	tempbs.read()
	addbits(bs,1,2)
	for dct in dcts:
		bs.write(dct)

def deltacomp(dcts,xdiff,bs):
	bs.read()
	tempbs.read()
	addbits(bs,2,2)
	wd = width//dc_chunk_size
	chunkdiffs = {}

	xdiff_flat = xdiff.flatten()
	for i in range(len(xdiff_flat)):
		if xdiff_flat[i]:
			x = i%width
			y = i//width
			idx = ((y//dc_chunk_size)*wd)+(x//dc_chunk_size)
			if idx not in chunkdiffs:
				chunkdiffs[idx] = 1
			else:
				chunkdiffs[idx] += 1

	addbits(bs,len(chunkdiffs)-1,dc_chunk_bits)
	for idx in chunkdiffs:
		addbits(bs,idx,dc_chunk_bits)
		bs.write(dcts[idx])

nbdup = 0
nbhuff = 0
nbdelta = 0
nboverh = 0
nboverd = 0
ratiohuff = 0
ratiodelta = 0

lastframe = np.zeros((height,width), np.uint8)

rlebs = BitStream()
deltabs = BitStream()

cframe = 0
encodedframes = 0

starttime = time.time()

addbits(gbits,int(fps*1000),16)

def add_freq(freqt, symb):
	if "tot" not in freqt:
		freqt["tot"] = 1
	else:
		freqt["tot"] += 1
	if symb in freqt:
		freqt[symb] += 1
	else:
		freqt[symb] = 1
#(0, 0),
zigzag = [(0, 1), (1, 0),
		(2, 0), (1, 1), (0, 2),
		(0, 3), (1, 2), (2, 1), (3, 0),
		(4, 0), (3, 1), (2, 2), (1, 3), (0, 4),
		(0, 5), (1, 4), (2, 3), (3, 2), (4, 1), (5, 0),
		(6, 0), (5, 1), (4, 2), (3, 3), (2, 4), (1, 5), (0, 6),
		(0, 7), (1, 6), (2, 5), (3, 4), (4, 3), (5, 2), (6, 1), (7, 0),
		(7, 1), (6, 2), (5, 3), (4, 4), (3, 5), (2, 6), (1, 7),
		(2, 7), (3, 6), (4, 5), (5, 4), (6, 3), (7, 2),
		(7, 3), (6, 4), (5, 5), (4, 6), (3, 7),
		(4, 7), (5, 6), (6, 5), (7, 4),
		(7, 5), (6, 6), (5, 7),
		(6, 7), (7, 6),
		(7, 7)]

def do_dct(chunk):
	quantized = np.rint(cv2.dct(chunk.astype(np.single))/quant).astype(np.int8)
	output = []
	nbzeros = 0
	lastval = 256
	lastnb = 0
	for pos in zigzag:
		val = quantized[pos[0]][pos[1]]
		if val != lastval:
			while lastnb >= 16:
				lastnb -= 15
				if lastval == 0:
					output.append((0xF0,0))
				else:
					output.append((0xF | ((nbzeros & 0xF) << 4),lastval))
					nbzeros = 0
			if lastval != 256 and lastval != 0:
				output.append(((lastnb & 0xF) | ((nbzeros & 0xF) << 4),lastval))
				nbzeros = 0
			elif lastval == 0:
				nbzeros = lastnb
			lastnb = 1
			lastval = val
		else:
			lastnb += 1
	if lastnb != 0:
		if lastval == 0:
			output.append((0,0))
		else:
			output.append(((lastnb & 0xF) | ((nbzeros & 0xF) << 4),lastval))
	return quantized[0][0],output,quantized

print("Applying DCT & Generating frequency table")
precalc_frames = []
precalc_frames_dct = {}
lastframeflat = np.zeros((height,width), np.uint8)
freqframes = 0
while vid.isOpened():
	r,f = vid.read()
	if not r:
		break
	f = img2bin(f)
	idx = len(precalc_frames)
	precalc_frames.append(f)
	if not (f ^ lastframeflat).any():
		continue
	print(CLEAR+"Frame: %i/%i (%.2f%%)"%(freqframes,nbframes,(freqframes/nbframes)*100),end='\r')

	wd = width//dc_chunk_size
	hd = height//dc_chunk_size

	chunks = view_as_blocks(f,(dc_chunk_size,dc_chunk_size))

	if cmd_args.show:
		dcto = []

		temph = []
		for linechunks in chunks:
			for i,chunk in enumerate(linechunks):
				dct = np.rint(cv2.dct(chunk.astype(np.single))/quant)
				idct = np.clip(cv2.dct(dct*quant,flags=cv2.DCT_INVERSE),-128,127)
				cout = np.rint(idct+128).astype(np.uint8)
				if cmd_args.debug: # and (i%wd) == wd-1:
					print("--------------------")
					print("Chunk:")
					print(chunk)
					print("DCT:")
					print(dct)
					print("IDCT:")
					print(idct)
					print("IDCT RINT:")
					print(np.rint(idct))
					print("Out:")
					print(cout)
				#cout = (np.rint(cv2.dct((np.rint(cv2.dct(chunk.astype(np.single))/quant).astype(np.int8).astype(np.single))*quant,flags=cv2.DCT_INVERSE))+127).astype(np.uint8)
				temph.append(cout)

			dcto.append(np.hstack(temph))
			temph.clear()
		dcto = np.vstack(dcto).astype(np.uint8)

		img = np.concatenate(((f.astype(np.int16)+128).astype(np.uint8),dcto),axis=1)
		cv2.imshow("Image",cv2.resize(img,(width*4*2,height*4),interpolation=cv2.INTER_NEAREST))
		waittime = 0
		if cmd_args.play:
			waittime = round((1/fps)*1000)
		if cv2.waitKey(waittime) == 27:
			cv2.destroyAllWindows()
			exit()

	precalc_frames_dct[idx] = []
	for linechunks in chunks:
		for chunk in linechunks:
			dct = do_dct(chunk)
			precalc_frames_dct[idx].append(dct)
			add_freq(valfrequency,dct[0])
			for i in dct[1]:
				add_freq(frequency,i[0])
				if i[0] != 0xF0 and i[0] != 0:
					add_freq(valfrequency,i[1])

	freqframes += 1
	lastframeflat = f

def get_smallest(nodes):
	s = 1
	n = -1
	for i,k in enumerate(nodes):
		if k["p"] != -1:
			continue
		if k["f"] < s:
			n = i
			s = k["f"]
	return n

# Otherwise C decompressor won't have same tree
tnparr = np.array((1,),dtype=np.single)
def addfloat(f1,f2):
	tnparr[0] = 0
	tnparr[0] += f1
	tnparr[0] += f2
	return tnparr[0]

def huffman(freqs):
	nodes = []
	for i in sorted(freqs, key=freqs.get):
		nodes.append({"s":i,"f":freqs[i],"p":-1,"l":-1,"r":-1})
	rootnode = -1
	while True:
		nm = len(nodes)
		n1 = get_smallest(nodes)
		if n1 == -1:
			rootnode = len(nodes)-1
			break
		nodes[n1]["p"] = nm
		n2 = get_smallest(nodes)
		if n2 == -1:
			nodes[n1]["p"] = -1
			nodes[n1]["f"] = 1
			rootnode = n1
			break
		nodes[n2]["p"] = nm
		nodes.append({"s":0,"f":addfloat(nodes[n1]["f"],nodes[n2]["f"]),"p":-1,"l":n1,"r":n2})
	table = {}
	for i in range(len(freqs)):
		vals = []
		nidx = i
		while True:
			nidxn = nodes[nidx]["p"]
			n = nodes[nidxn]
			if n["l"] == nidx:
				vals.insert(0,False)
			elif n["r"] == nidx:
				vals.insert(0,True)
			if n["p"] == -1:
				break
			nidx = nidxn
		table[nodes[i]["s"]] = vals
	return table

def to_unsigned(val):
	if val < 0:
		return val+256
	return val

def add_table(freqs,bs):
	total = freqs["tot"]
	del freqs["tot"]
	addbits(bs,len(freqs)-1,8)
	for i in sorted(freqs, key=freqs.get):
		freqs[i] = freqs[i]/total
		packed = struct.pack("<f",freqs[i])
		# Make sure same value as in decoder (truncate precision), just in case
		freqs[i] = struct.unpack("<f",packed)[0]

	for i in sorted(freqs, key=freqs.get):
		addbits(bs,to_unsigned(i),8)
		bs.write(struct.pack("<f",freqs[i]))

	return huffman(freqs)

def dct_huffman(dct):
	out = []
	out.extend(valsymboltable[dct[0]])
	for i in dct[1]:
		out.extend(symboltable[i[0]])
		if i[0] != 0xF0 and i[0] != 0:
			out.extend(valsymboltable[i[1]])
	return out

for row in quant:
	for item in row:
		addbits(gbits,item,16)

symboltable = add_table(frequency,gbits)
valsymboltable = add_table(valfrequency,gbits)
"""
def mkst(vals):
	st = ""
	for i in vals:
		if i:
			st += "1"
		else:
			st += "0"
	return st

print("")
print("")
print("Marker:")
for i in sorted(frequency, key=frequency.get):
	print(to_unsigned(i),mkst(symboltable[i]))
print("Values:")
for i in sorted(valfrequency, key=valfrequency.get):
	print(to_unsigned(i),mkst(valsymboltable[i]))
print("")
"""

print(CLEAR+"Done marker:%i,values:%i symbols"%(len(frequency),len(valfrequency)))

for idx,f in enumerate(precalc_frames):
	cframe += 1
	fps = cframe/(time.time()-starttime)
	#if cframe%10 == 0:
	print(CLEAR+"Frame: %i/%i (%.2f%%) %is remaining"%(cframe,nbframes,(cframe/nbframes)*100,(nbframes-cframe)/fps),end='\r')

	diff = f ^ lastframe
	if not diff.any():
		nbdup += 1
		addbits(gbits,3,2) # Duplicate frame
		continue

	frame_size = 2+width*height*8

	dcts = [dct_huffman(dct) for dct in precalc_frames_dct[idx]]

	huffcomp(dcts,rlebs)
	deltacomp(dcts,diff,deltabs)

	if len(rlebs) >= frame_size:
		nboverh += 1
	if len(deltabs) >= frame_size:
		nboverd += 1

	ratiohuff += (len(rlebs)-2)/(frame_size-2)
	ratiodelta += (len(deltabs)-2)/(frame_size-2)
	encodedframes += 1

	if len(rlebs) <= len(deltabs):
		gbits.write(rlebs)
		nbhuff += 1
	else:
		gbits.write(deltabs)
		nbdelta += 1

	lastframe = f

endtime = time.time()

nbbits = len(gbits)
if nbbits%8 != 0:
	for i in range(8-(nbbits%8)):
		gbits.write(False)

size = 0
with open(outfile,"wb") as ofile:
	ofile.write(gbits.read(bytes))
	size = ofile.tell()

print("")
print("----------")
print("Number of frames: %i"%(cframe))
print("Huff frames     : %i (%.2f%%)"%(nbhuff,(nbhuff/nbframes)*100))
print("Delta frames    : %i (%.2f%%)"%(nbdelta,(nbdelta/nbframes)*100))
print("Duplicate frames: %i (%.2f%%)"%(nbdup,(nbdup/nbframes)*100))
print("----------")
print("Huff frames over      : %i (%.2f%%)"%(nboverh,(nboverh/encodedframes)*100))
print("Delta frames over     : %i (%.2f%%)"%(nboverd,(nboverd/encodedframes)*100))
print("Huff frames avg ratio : %.2f%%"%((1-(ratiohuff/encodedframes))*100))
print("Delta frames avg ratio: %.2f%%"%((1-(ratiodelta/encodedframes))*100))
print("----------")
print("Normal    : %i bytes"%normal_size)
print("Compressed: %i bytes"%size)
print("Ratio     : %.2f%%"%((1-(size/normal_size))*100))
print("Speed     : %.2ffps"%(nbframes/(endtime-starttime)))


from parsivar import FindStems, Tokenizer, Normalizer, normalizer
import re
from read_contents import read_contents, Counter
from sklearn.feature_selection import chi2, SelectKBest

file = open('StopWords', 'r', encoding='UTF-8')
lines = file.readlines()
stop_words_list = []
for line in lines:
    stop_words_list.append(line.rstrip('\n'))
badChars = ['.', '!', '?', ':', ';', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
            'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D', 'E',
            'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W',
            'X', 'Y', 'Z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '&', '<', '>', '=', '-',
            ]
emails1 = "Spam-Filtering-For-Persian-master/emails/spamtraining"
emails2 = "Spam-Filtering-For-Persian-master/emails/hamtraining"

transformer = SelectKBest().fit()
# def preprocess_words(address, index):
#     email = read_contents(address, index)
#     string = ""
#     l = []
#     for i in email:
#         if i != '' and i != '\n':
#             string = string + i
#             l.append(i)
#     for sentence in l:
#         sentence.replace("/", "")
#     string.replace("/", "")
#     pattern = '[' + ''.join(badChars) + ']'
#     string = re.sub(pattern, '', string)
#     my_normalizer = Normalizer()
#     my_tokenizer = Tokenizer()
#     words = my_tokenizer.tokenize_words(my_normalizer.normalize(string))
#     return l


"""
主要方法
"""
import requests
import json
import xlwt

import tools

# 获取各页的视频bvid
def getPagesBvId(mid):
    # 初始化变量
    videoDataLists = []
    index = 0
    pages = 0
    # 循环取bvid
    while True:
        # 测试数据：https://api.bilibili.com/x/space/arc/search?mid=691415738&jsonp=jsonp&pn=1
        pageURL = "https://api.bilibili.com/x/space/arc/search?mid=" + mid + "&jsonp=jsonp&pn=" + str(index + 1)
        videoDataList = grab_info(pageURL)
        # 如果是第一页，就去判断是否有数据，顺便获取总共有几页视频和作者信息
        if index == 0:
            if videoDataList == "":
                tools.error("【error】：mid为【" + mid + "】的UP没有视频，请重新选择操作！")
                return False
            elif videoDataList == "null":
                tools.error("【error】：查不到mid为：【" + mid + "】的UP数据，请重新选择操作！")
                return False
            # 计算该UP总共有多少个视频
            pages = int(
                (videoDataList['page']['count'] + videoDataList['page']['ps'] - 1) / videoDataList['page']['ps'])
            author = videoDataList['list']['vlist'][0]['author']
            print("【" + author + "】总共有", pages, "页视频")
        # 如果是最后一页 则去计算最后一页有多少个视频
        if videoDataList['page']['pn'] == pages:
            videoDataList['page']['ps'] = videoDataList['page']['count'] - (videoDataList['page']['pn'] - 1) * 30
        # 获取视频数据，添加到videoDataLists
        for i in range(videoDataList['page']['ps']):
            videoDataLists.append(videoDataList['list']['vlist'][i])
        print("第", index + 1, "页:", pageURL, "完成")
        # 累加页码
        index += 1
        # 当最后一页执行完之后 退出循环
        if index == pages:
            break
    return videoDataLists

# 获取地址的json信息
def grab_info(FullURL):
    # 代理头部
    headers = {
        "User-Agent": tools.getRandomUserAgent()
    }
    # 获取一个返回信息
    response = requests.get(FullURL, headers=headers).content.decode('utf-8')
    # 把json转换为python对象
    content = json.loads(response)
    # 利用json提取主体视频信息
    if content['code'] == 0:
        data = content['data']
        if len(data['list']['vlist']) == 0:
            return ""
    else:
        return "null"
    return data

# 根据list生成excel文件
def createExcel(dateList, functionName):
    workbook = xlwt.Workbook(encoding="utf-8")  # 实例化book对象
    sheet = workbook.add_sheet(tools.getSheetName(dateList, functionName))  # 生成sheet

    # 设置excel的title、数据列表中的key和列宽
    titleList = tools.getTitleList(functionName)[0]
    listKey = tools.getTitleList(functionName)[1]
    titleWidth = tools.getTitleList(functionName)[2]

    # 设置title宽度
    for i in range(len(titleWidth)):
        sheet.col(i).width = titleWidth[i]

    # 写入标题
    for col, column in enumerate(titleList):
        sheet.write(0, col, column)

    # 写入数据
    for row, data in enumerate(dateList):
        for i in range(len(listKey)):
            # 0为序号列 i>0为数据列
            if i == 0:
                sheet.write(row + 1, i, row + 1)
            else:
                if listKey[i] == 'created':
                    # 如果该列为created:上传时间,需要将时间戳转换为时间
                    sheet.write(row + 1, i, tools.timeStampToTime(data[listKey[i]]))
                else:
                    sheet.write(row + 1, i, data[listKey[i]])
    workbook.save(tools.getExcelName(dateList, functionName))

# 创建bat文件
def createBat(dataList, functionName):
    # TODO 通过配置文件读取cd地址
    cdCommand = "CD D:/Project/PyCharm/bilibiliDown/downloader\nD:\n"
    downCommand = tools.getDownCommand(dataList, functionName)

    # 生成bat文件 common.getBatName(dateList, functionName)=>获取bat文件名
    with open(tools.getBatName(dataList, functionName) + ".bat", 'w', encoding='utf-8') as f:
        f.write(cdCommand + downCommand)
    f.close()


import pygame as pg

scrWidth = 600
scrHeight = 400
win = pg.display.set_mode((scrWidth,scrHeight))

pg.display.set_caption("First Game")

# This goes outside the while loop, near the top of the program
walkRight = [pg.image.load('R1.png'), pg.image.load('R2.png'), pg.image.load('R3.png'), pg.image.load('R4.png'), pg.image.load('R5.png'), pg.image.load('R6.png')]
walkLeft = [pg.image.load('L1.png'), pg.image.load('L2.png'), pg.image.load('L3.png'), pg.image.load('L4.png'), pg.image.load('L5.png'), pg.image.load('L6.png')]
background = pg.image.load('background.png')
char = pg.image.load('R1.png')

clock = pg.time.Clock()

x = 50
y = 350
chrWidth = 17
chrHeight = 27
vel = 5
isJump = False
jumpCount = 10
left = False
right = False
direction = 'right'
walkCount = 0

def redrawWindow():
    global walkCount, direction

    win.blit(background, (0,0))

    if walkCount + 1 >= 18:
        walkCount = 0

    if left == True:
        win.blit(walkLeft[walkCount // 3], (x,y))
        walkCount += 1
        direction = 'left'
    elif right == True:
        win.blit(walkRight[walkCount // 3], (x,y))
        walkCount += 1
        direction = 'right'
    else:
        if direction == 'right':
            win.blit(walkRight[0], (x,y))
        elif direction == 'left':
            win.blit(walkLeft[0], (x,y))

    pg.display.update()

# main loop
run = True
while run:
    clock.tick(18)

    for event in pg.event.get():
        if event.type == pg.QUIT:
            run = False

    keys = pg.key.get_pressed()

    if keys[pg.K_LEFT] and x > vel:
        x -= vel
        left = True
        right = False
    elif keys[pg.K_RIGHT] and x < scrWidth - chrWidth - vel:
        x += vel
        right = True
        left = False
    else:
        right = False
        left = False
        walkCount = 0

    if isJump == False:
        if keys[pg.K_SPACE]:
            isJump = True
            right = False
            left = False
            walkCount = 0
    elif isJump == True:
        if jumpCount >= 0:
            y -= (jumpCount ** 2) * 0.35
            jumpCount -= 1
        elif jumpCount < 0:
            y += (jumpCount ** 2) * 0.35
            jumpCount -= 1
        if jumpCount < -10:
            isJump = False
            jumpCount = 10

    redrawWindow()

pg.quit()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('students', '0001_initial'),
    ]

    operations = [
        migrations.AlterField(
            model_name='tests',
            name='exam_title',
            field=models.CharField(max_length=256, null=True, verbose_name='\u041d\u0430\u0437\u0432\u0430 \u0435\u043a\u0437\u0430\u043c\u0435\u043d\u0443 '),
        ),
        migrations.AlterField(
            model_name='tests',
            name='notes',
            field=models.TextField(null=True, verbose_name='\u041d\u043e\u0442\u0430\u0442\u043a\u0438', blank=True),
        ),
        migrations.AlterField(
            model_name='tests',
            name='teacher',
            field=models.CharField(max_length=256, null=True, verbose_name='\u0412\u0438\u043a\u043b\u0430\u0434\u0430\u0447'),
        ),
    ]


import time
import logging
import functools

import numpy as np
from qiskit import ClassicalRegister, QuantumCircuit

from qiskit.aqua.algorithms import QuantumAlgorithm
from qiskit.aqua import AquaError, PluggableType, get_pluggable_class
from qiskit.aqua.utils import find_regs_by_name
from qiskit.aqua.utils.backend_utils import is_aer_statevector_backend

logger = logging.getLogger(__name__)

class VQE(QuantumAlgorithm):
    """
    The Variational Quantum Eigensolver algorithm.

    See https://arxiv.org/abs/1304.3061
    """

    CONFIGURATION = {
        'name': 'VQE',
        'description': 'VQE Algorithm',
        'input_schema': {
            '$schema': 'http://json-schema.org/schema#',
            'id': 'vqe_schema',
            'type': 'object',
            'properties': {
                'operator_mode': {
                    'type': 'string',
                    'default': 'matrix',
                    'oneOf': [
                        {'enum': ['matrix', 'paulis', 'grouped_paulis']}
                    ]
                },
                'initial_point': {
                    'type': ['array', 'null'],
                    "items": {
                        "type": "number"
                    },
                    'default': None
                },
                'batch_mode': {
                    'type': 'boolean',
                    'default': False
                }
            },
            'additionalProperties': False
        },
        'problems': ['energy', 'ising'],
        'depends': ['optimizer', 'variational_form', 'initial_state'],
        'defaults': {
            'optimizer': {
                'name': 'L_BFGS_B'
            },
            'variational_form': {
                'name': 'RYRZ'
            },
            'initial_state': {
                'name': 'ZERO'
            }
        }
    }

    def __init__(self, operator, var_form, optimizer, operator_mode='matrix',
                 initial_point=None, batch_mode=False, aux_operators=None):
        """Constructor.

        Args:
            operator (Operator): Qubit operator
            operator_mode (str): operator mode, used for eval of operator
            var_form (VariationalForm): parametrized variational form.
            optimizer (Optimizer): the classical optimization algorithm.
            initial_point (numpy.ndarray): optimizer initial point.
            batch_mode (bool): evaluate parameter sets in parallel.
            aux_operators (list of Operator): Auxiliary operators to be evaluated at each eigenvalue
        """
        self.validate(locals())
        super().__init__()
        self._operator = operator
        self._var_form = var_form
        self._optimizer = optimizer
        self._operator_mode = operator_mode
        self._initial_point = initial_point
        if initial_point is None:
            self._initial_point = var_form.preferred_init_points
        self._optimizer.set_batch_mode(batch_mode)
        if aux_operators is None:
            self._aux_operators = []
        else:
            self._aux_operators = [aux_operators] if not isinstance(aux_operators, list) else aux_operators
        self._ret = {}
        self._eval_count = 0
        self._eval_time = 0
        logger.info(self.print_setting())

    @classmethod
    def init_params(cls, params, algo_input):
        """
        Initialize via parameters dictionary and algorithm input instance.

        Args:
            params (dict): parameters dictionary
            algo_input (EnergyInput): EnergyInput instance

        Returns:
            VQE: vqe object
        """
        if algo_input is None:
            raise AquaError("EnergyInput instance is required.")

        operator = algo_input.qubit_op

        vqe_params = params.get(QuantumAlgorithm.SECTION_KEY_ALGORITHM)
        operator_mode = vqe_params.get('operator_mode')
        initial_point = vqe_params.get('initial_point')
        batch_mode = vqe_params.get('batch_mode')

        # Set up initial state, we need to add computed num qubits to params
        init_state_params = params.get(QuantumAlgorithm.SECTION_KEY_INITIAL_STATE)
        init_state_params['num_qubits'] = operator.num_qubits
        init_state = get_pluggable_class(PluggableType.INITIAL_STATE,
                                         init_state_params['name']).init_params(init_state_params)

        # Set up variational form, we need to add computed num qubits, and initial state to params
        var_form_params = params.get(QuantumAlgorithm.SECTION_KEY_VAR_FORM)
        var_form_params['num_qubits'] = operator.num_qubits
        var_form_params['initial_state'] = init_state
        var_form = get_pluggable_class(PluggableType.VARIATIONAL_FORM,
                                       var_form_params['name']).init_params(var_form_params)

        # Set up optimizer
        opt_params = params.get(QuantumAlgorithm.SECTION_KEY_OPTIMIZER)
        optimizer = get_pluggable_class(PluggableType.OPTIMIZER,
                                        opt_params['name']).init_params(opt_params)

        return cls(operator, var_form, optimizer, operator_mode=operator_mode,
                   initial_point=initial_point, batch_mode=batch_mode,
                   aux_operators=algo_input.aux_ops)

    @property
    def setting(self):
        """Prepare the setting of VQE as a string."""
        ret = "Algorithm: {}\n".format(self._configuration['name'])
        params = ""
        for key, value in self.__dict__.items():
            if key != "_configuration" and key[0] == "_":
                if "initial_point" in key and value is None:
                    params += "-- {}: {}\n".format(key[1:], "Random seed")
                else:
                    params += "-- {}: {}\n".format(key[1:], value)
        ret += "{}".format(params)
        return ret

    def print_setting(self):
        """
        Preparing the setting of VQE into a string.

        Returns:
            str: the formatted setting of VQE
        """
        ret = "\n"
        ret += "==================== Setting of {} ============================\n".format(self.configuration['name'])
        ret += "{}".format(self.setting)
        ret += "===============================================================\n"
        ret += "{}".format(self._var_form.setting)
        ret += "===============================================================\n"
        ret += "{}".format(self._optimizer.setting)
        ret += "===============================================================\n"
        return ret

    def construct_circuit(self, parameter, backend=None, use_simulator_operator_mode=False):
        """Generate the circuits.

        Args:
            parameters (numpy.ndarray): parameters for variational form.
            backend (qiskit.BaseBackend): backend object.
            use_simulator_operator_mode (bool): is backend from AerProvider, if True and mode is paulis,
                           single circuit is generated.

        Returns:
            [QuantumCircuit]: the generated circuits with Hamiltonian.
        """
        input_circuit = self._var_form.construct_circuit(parameter)
        if backend is None:
            warning_msg = "Circuits used in VQE depends on the backend type, "
            from qiskit import BasicAer
            if self._operator_mode == 'matrix':
                temp_backend_name = 'statevector_simulator'
            else:
                temp_backend_name = 'qasm_simulator'
            backend = BasicAer.get_backend(temp_backend_name)
            warning_msg += "since operator_mode is '{}', '{}' backend is used.".format(
                self._operator_mode, temp_backend_name)
            logger.warning(warning_msg)
        circuit = self._operator.construct_evaluation_circuit(self._operator_mode,
                                                              input_circuit, backend, use_simulator_operator_mode)
        return circuit

    def _solve(self):
        opt_params, opt_val = self.find_minimum_eigenvalue()
        self._ret['eigvals'] = np.asarray([opt_val])
        self._ret['opt_params'] = opt_params
        qc = self._var_form.construct_circuit(self._ret['opt_params'])
        if self._quantum_instance.is_statevector:
            ret = self._quantum_instance.execute(qc)
            self._ret['eigvecs'] = np.asarray([ret.get_statevector(qc)])
        else:
            c = ClassicalRegister(self._operator.num_qubits, name='c')
            q = find_regs_by_name(qc, 'q')
            qc.add_register(c)
            qc.barrier(q)
            qc.measure(q, c)
            ret = self._quantum_instance.execute(qc)
            self._ret['eigvecs'] = np.asarray([ret.get_counts(qc)])

    def _get_ground_state_energy(self):
        if 'eigvals' not in self._ret:
            self._solve()
        self._ret['energy'] = self._ret['eigvals'][0]

    def _eval_aux_ops(self, threshold=1e-12):
        if 'opt_params' not in self._ret:
            self._get_ground_state_energy()
        wavefn_circuit = self._var_form.construct_circuit(self._ret['opt_params'])
        circuits = []
        values = []
        params = []
        for operator in self._aux_operators:
            if not operator.is_empty():
                temp_circuit = QuantumCircuit() + wavefn_circuit
                circuit = operator.construct_evaluation_circuit(self._operator_mode, temp_circuit,
                                                                self._quantum_instance.backend,
                                                                self._use_simulator_operator_mode)
                params.append(operator.aer_paulis)
            else:
                circuit = None
            circuits.append(circuit)

        if len(circuits) > 0:
            to_be_simulated_circuits = functools.reduce(lambda x, y: x + y, [c for c in circuits if c is not None])
            if self._use_simulator_operator_mode:
                extra_args = {'expectation': {
                    'params': params,
                    'num_qubits': self._operator.num_qubits}
                }
            else:
                extra_args = {}
            result = self._quantum_instance.execute(to_be_simulated_circuits, **extra_args)

            for operator, circuit in zip(self._aux_operators, circuits):
                if circuit is None:
                    mean, std = 0.0, 0.0
                else:
                    mean, std = operator.evaluate_with_result(self._operator_mode,
                                                              circuit, self._quantum_instance.backend,
                                                              result, self._use_simulator_operator_mode)
                mean = mean.real if abs(mean.real) > threshold else 0.0
                std = std.real if abs(std.real) > threshold else 0.0
                values.append((mean, std))
        if len(values) > 0:
            aux_op_vals = np.empty([1, len(self._aux_operators), 2])
            aux_op_vals[0, :] = np.asarray(values)
            self._ret['aux_ops'] = aux_op_vals

    def _run(self):
        """
        Run the algorithm to compute the minimum eigenvalue.

        Returns:
            Dictionary of results
        """
        if not self._quantum_instance.is_statevector and self._operator_mode == 'matrix':
            logger.warning('Qasm simulation does not work on {} mode, changing '
                           'the operator_mode to "paulis"'.format(self._operator_mode))
            self._operator_mode = 'paulis'

        self._use_simulator_operator_mode = \
            is_aer_statevector_backend(self._quantum_instance.backend) \
            and self._operator_mode != 'matrix'

        self._quantum_instance.circuit_summary = True
        self._eval_count = 0
        self._solve()
        self._get_ground_state_energy()
        self._eval_aux_ops()
        self._ret['eval_count'] = self._eval_count
        self._ret['eval_time'] = self._eval_time
        return self._ret

    # This is the objective function to be passed to the optimizer that is uses for evaluation
    def _energy_evaluation(self, parameters):
        """
        Evaluate energy at given parameters for the variational form.

        Args:
            parameters (numpy.ndarray): parameters for variational form.

        Returns:
            float or list of float: energy of the hamiltonian of each parameter.
        """
        num_parameter_sets = len(parameters) // self._var_form.num_parameters
        circuits = []
        parameter_sets = np.split(parameters, num_parameter_sets)
        mean_energy = []
        std_energy = []

        for idx in range(len(parameter_sets)):
            parameter = parameter_sets[idx]
            circuit = self.construct_circuit(parameter, self._quantum_instance.backend, self._use_simulator_operator_mode)
            circuits.append(circuit)

        to_be_simulated_circuits = functools.reduce(lambda x, y: x + y, circuits)
        if self._use_simulator_operator_mode:
            extra_args = {'expectation': {
                'params': [self._operator.aer_paulis],
                'num_qubits': self._operator.num_qubits}
            }
        else:
            extra_args = {}
        result = self._quantum_instance.execute(to_be_simulated_circuits, **extra_args)

        for idx in range(len(parameter_sets)):
            mean, std = self._operator.evaluate_with_result(
                self._operator_mode, circuits[idx], self._quantum_instance.backend, result, self._use_simulator_operator_mode)
            mean_energy.append(np.real(mean))
            std_energy.append(np.real(std))
            self._eval_count += 1
            logger.info('Energy evaluation {} returned {}'.format(self._eval_count, np.real(mean)))

        return mean_energy if len(mean_energy) > 1 else mean_energy[0]

    def find_minimum_eigenvalue(self, initial_point=None):
        """Determine minimum energy state.

        Args:
            initial_point (numpy.ndarray[float]) : initial point, or None
                if not provided.

        Returns:
            Optimized variational parameters, and corresponding minimum eigenvalue.

        Raises:
            ValueError:

        """
        initial_point = initial_point if initial_point is not None else self._initial_point

        nparms = self._var_form.num_parameters
        bounds = self._var_form.parameter_bounds

        if initial_point is not None and len(initial_point) != nparms:
            raise ValueError('Initial point size {} and parameter size {} mismatch'.format(len(initial_point), nparms))
        if len(bounds) != nparms:
            raise ValueError('Variational form bounds size does not match parameter size')
        # If *any* value is *equal* in bounds array to None then the problem does *not* have bounds
        problem_has_bounds = not np.any(np.equal(bounds, None))
        # Check capabilities of the optimizer
        if problem_has_bounds:
            if not self._optimizer.is_bounds_supported:
                raise ValueError('Problem has bounds but optimizer does not support bounds')
        else:
            if self._optimizer.is_bounds_required:
                raise ValueError('Problem does not have bounds but optimizer requires bounds')
        if initial_point is not None:
            if not self._optimizer.is_initial_point_supported:
                raise ValueError('Optimizer does not support initial point')
        else:
            if self._optimizer.is_initial_point_required:
                low = [(l if l is not None else -2 * np.pi) for (l, u) in bounds]
                high = [(u if u is not None else 2 * np.pi) for (l, u) in bounds]
                initial_point = self.random.uniform(low, high)

        start = time.time()
        logger.info('Starting optimizer bounds={}\ninitial point={}'.format(bounds, initial_point))
        sol, opt, nfev = self._optimizer.optimize(self._var_form.num_parameters, self._energy_evaluation,
                                                  variable_bounds=bounds, initial_point=initial_point)
        if nfev is not None:
            self._eval_count = self._eval_count if self._eval_count >= nfev else nfev
        self._eval_time = time.time() - start
        logger.info('Optimization complete in {}s found {} num evals {}'.format(
            self._eval_time, opt, self._eval_count))

        return sol, opt