from django import template
from django.db.models.loading import cache
from django.db.models import Model as DjangoModel
from django.utils.safestring import SafeUnicode

from classytags.core import Options
from classytags.arguments import (Argument, MultiValueArgument, MultiKeywordArgument)
from classytags.helpers import AsTag

from .. import models

register = template.Library()

@register.filter
def minus(value, args):
	try:
		return int(value) - int(args)

	except ValueError as error:
		return error

@register.filter
def fields(instance, args=None):
	try:
		return instance._meta.get_all_field_names()

	except ValueError as error:
		return error

@register.tag
class ProjectsQuerySetTag(AsTag):
	name = "get_projects"
	options = Options(
		MultiKeywordArgument('kwargs', resolve=True, required=False),
		'as',
		Argument('varname', resolve=False, required=False),
	)

	def get_value(self, context, kwargs=None, varname=None):
		count = 4
		if kwargs != None:
			tag = kwargs.get("tag", None)
			phase = kwargs.get("phase", None)
			count = kwargs.get("count", count)

		results = models.Project.objects.all()
		if tag != None:
			results = results.filter(tags__name=tag)

		if phase != None:
			results = results.filter(phase=phase)

		results = results[:count]

		return results

@register.tag
class ClientsQuerySetTag(AsTag):
	name = "get_clients"
	options = Options(
		MultiKeywordArgument('kwargs', resolve=True, required=False),
		'as',
		Argument('varname', resolve=False, required=False),
	)

	def get_value(self, context, kwargs=None, varname=None):
		count = 4
		if kwargs != None:
			tags = kwargs.get("tags", None)
			nottags = kwargs.get("nottags", None)
			phase = kwargs.get("phase", None)
			count = kwargs.get("count", count)

		results = models.Client.objects.all()

		if tags != None:
			if type(tags) == str or type(tags) == SafeUnicode:
				tags = tags.split(",")
			results = results.filter(tags__name__in=tags)

		if nottags != None:
			if type(nottags) == str or type(nottags) == SafeUnicode:
				nottags = nottags.split(",")
			results = results.exlude(tags__name__in=nottags)

		if type(count) == int:
			results = results[:count]
		return results


import jwt
from flask_httpauth import HTTPTokenAuth
from sqlalchemy import select

from app import db, secret_token
from app.daos.user_dao import check_password
from app.models.user import User

token_auth = HTTPTokenAuth(scheme="Bearer")

@token_auth.verify_token
def verify_token(token):
    try:
        decoded_jwt = jwt.decode(token, secret_token, algorithms=["HS256"])
    except Exception:
        print("Unable to decode JWT")
        return None

    username = decoded_jwt["username"]
    user = db.session.scalars(
        select(User).where(User.username == username)
    ).one_or_none()
    if user:
        return user
    print(f"Unknown user {username}")
    return None


# LeetCode Problem Nr. 34: 反转字符串

from typing import List

class Solution:
    def reverseString(self, s: List[str]) -> None:
        return self.__method1(s)

    def __method1(self, s: List[str]) -> None:
        left, right = 0, len(s) - 1
        while left < right:
            s[left], s[right] = s[right], s[left]
            left += 1
            right -= 1


from tkinter import *
import tkinter as tk
import playsound
from PIL import ImageTk, Image, ImageSequence
import threading
import time
import random
import sys

def disable_event():
    pass

def YouAreAnIdiot():
    global img

    root= Tk()
    root.title("Idiot")
    # root.attributes('-toolwindow', True) # bad attribute "-toolwindow": must be -alpha, -topmost, -zoomed, -fullscreen, or -type

    # play a sound file
    playsound.playsound('idiot.mp3', True)

    # root attributes: -alpha, -topmost, -zoomed, -fullscreen, -type
    if sys.platform.startswith('win32') or sys.platform.startswith('cygwin'):
        root.attributes('-toolwindow', True)
    else:
        root.attributes('-alpha', 0.5)

    x= _getRandomX()
    y= _getRandomY()
    # print('this is x: ' + str(x))
    # print('this is y: ' + str(y))
    root.resizable(0,0)
    root.geometry(f'234x171+{x}+{y}')

    # change background color in every new window

    color = '#%06x' % random.randint(0, 0xFFFFFF)
    root.configure(background=color)

    # Label(root, text="Idiot", font=("Helvetica", 20),background=color ).pack()
    # Label(root, text="You are an idiot", font=("Helvetica", 20),  ).pack()
    # no background color font

    Label(root, text="You are an idiot", font=("Franklin Gothic Medium Cond", 23),background=color, anchor='center' ).pack(
        fill=BOTH, expand=1)

    root.protocol("WM_DELETE_WINDOW", disable_event)

    root.mainloop()

values_of_x = []
values_of_y = []
def _getRandomX():
    global values_of_x
    # print(values_of_x)
    x = random.randint(0,999)
    if x not in values_of_x:
        values_of_x.append(x)
        return x
    else:
        return _getRandomX()

def _getRandomY():
    global values_of_y
    # print(values_of_y)
    y = random.randint(0,999)
    if y not in values_of_y:
        values_of_y.append(y)
        return y
    else:
        return _getRandomY()

if __name__=='__main__':
    while True:
        thread = threading.Thread(target=YouAreAnIdiot)
        thread.start()
        time.sleep(0.6)


from datetime import datetime
import uuid
from typing import Optional, Union

from pydantic import BaseModel

class BillingBase(BaseModel):
    id: uuid.UUID

    class Config:
        orm_mode = True

class DateTimeBase(BillingBase):
    created_at: Optional[datetime]
    updated_at: Optional[datetime]

class SubscriptionPlanSchema(BillingBase):
    price: int
    duration: int
    currency: str
    status: str

    class Config:
        orm_mode = True

class SubscriptionSchema(DateTimeBase):
    customer_id: uuid.UUID
    plan_id: uuid.UUID
    start_date: datetime
    end_date: datetime
    status: str
    payment_id: str

    class Config:
        orm_mode = True

class RefundSchema(DateTimeBase):
    payment_id: str
    customer_id: Union[str, uuid.UUID]
    amount: int
    status: str

    class Config:
        orm_mode = True

class CreateSubscriptionSchema(BaseModel):
    subscription_plan_id: str


#!/usr/bin/env python3

# std import
import csv
import sys
import argparse

# pip import
import itertools
import networkx as nx

def main(args=None):

    if args is None:
        args = sys.argv[1:]

    parser = argparse.ArgumentParser(prog="knot.analysis.classifications")

    parser.add_argument("-i", "--input", type=argparse.FileType('r'), help="path to the AAG", required=True)
    parser.add_argument("-o", "--output", type=argparse.FileType('w'), help="path where hamilton path was write", required=True)
    parser.add_argument("-c", "--circular", action="store_true", help="genome is circular")

    args = vars(parser.parse_args(args))

    graph = AAG2graph(args["input"])

    paths = []
    for cycle in valid_cycle(graph, args["circular"]):
        weight = sum([graph.edges[edge[0], edge[1]]["weight"] for edge in cycle])

        nodes = ",".join(remove_duplicate_if_follow([e for edge in cycle for e in edge]))
        paths.append((weight, nodes))

    paths.sort()

    for (weight, nodes) in paths:
        print(";".join([nodes, str(weight)]), file=args["output"])

def remove_duplicate_if_follow(l):
    previous = None
    for elt in l:
        if elt == previous:
            previous = elt
            continue

        previous = elt
        yield elt

def valid_cycle(graph, circular_genome=False):
    for g in itertools.permutations(graph.nodes()):
        if not contig_congruence(g):
            continue

        if g.index(min(g)) != 0:
            continue

        if circular_genome:
            edges = [edge for edge in zip(g, g[1:] + g[:1])]
        else:
            edges = [edge for edge in zip(g, g[1:])]

        if any([edge not in graph.edges for edge in edges]) :
            continue

        yield edges

def contig_congruence(g):
    for i, j in zip(g[::2], g[1::2]):
        if i.split("_")[0] != j.split("_")[0]:
            return False
    return True

def AAG2graph(AAG_path):

    paths = dict()
    reader = csv.DictReader(AAG_path)
    for row in reader:
        key = frozenset((row["tig1"], row["tig2"]))

        if row["paths"] == "not_found":
            continue

        if row["paths"] == "not_search":
            nb_base = 0
        else:
            nb_base = int(row["nb_base"])

        if nb_base < 0:
            nb_base = 0

        if key in paths and paths[key] < nb_base:
            continue

        paths[key] = nb_base

    graph = nx.DiGraph()
    for (tig1, tig2), nb_base in paths.items():
        graph.add_edge(tig1, tig2, weight=nb_base)
        graph.add_edge(tig2, tig1, weight=nb_base)

    tig_in_graph = set()
    for key in paths.keys():
        for k in key:
            tig_in_graph.add(k.split("_")[0])

    for tig in tig_in_graph:
        graph.add_edge(tig+"_begin", tig+"_end", weight=0)
        graph.add_edge(tig+"_end", tig+"_begin", weight=0)

    return graph

if __name__ == "__main__":
    main()


from config import hparams

PAD = "<pad>"
EOS = "<EOS>"
UNK = "<UNK>"
SEG = "|"
RESERVED_TOKENS = [PAD, EOS, UNK]
NUM_RESERVED_TOKENS = len(RESERVED_TOKENS)
PAD_ID = RESERVED_TOKENS.index(PAD)  # Normally 0
EOS_ID = RESERVED_TOKENS.index(EOS)  # Normally 1
UNK_ID = RESERVED_TOKENS.index(UNK)  # Normally 2

_initialized = False
_ALL_CONSONANTS_SET = set()
_ALL_VOWELS_SET = set()

#读入音素表
def load_phoneme_list(phoneme_path):
    _g2p_dictionary = {
        'AP': ['AP'],
        'SP': ['SP']
    }
    _set = set()
    #tmp_file_path = os.getcwd()
    #tmp_file_path = tmp_file_path + '/' + phoneme_path
    with open(phoneme_path, 'r', encoding='utf8') as _df:
        _lines = _df.readlines()
    for _line in _lines:
        _pinyin, _ph_str = _line.strip().split('\t')
        _g2p_dictionary[_pinyin] = _ph_str.split()
    for _list in _g2p_dictionary.values():
        [_set.add(ph) for ph in _list]
    _phoneme_list = sorted(list(_set))
    print('| load phone set:', _phoneme_list)

    phone_id = dict()
    id_phoneme = dict()
    for res_token in RESERVED_TOKENS:
        phone_id [ res_token ] = RESERVED_TOKENS.index(res_token)
        id_phoneme [RESERVED_TOKENS.index(res_token)] = res_token

    index_begin = len(RESERVED_TOKENS)
    for phoneme in _phoneme_list:
        phone_id[phoneme] = index_begin
        id_phoneme[index_begin] = phoneme
        index_begin = index_begin + 1

    return _phoneme_list,_g2p_dictionary,phone_id,id_phoneme,

def get_all_vowels():
    global _initialized
    if not _initialized:
        # 读入音素字典
        phone_list, pinyin2phs, phone_id, id_phoneme = load_phoneme_list(hparams['g2p_dictionary'])
        for _ph_list in pinyin2phs.values():
            _ph_count = len(_ph_list)
            if _ph_count == 0 or _ph_list[0] in ['AP', 'SP']:
                continue
            elif len(_ph_list) == 1:
                _ALL_VOWELS_SET.add(_ph_list[0])
            else:
                _ALL_CONSONANTS_SET.add(_ph_list[0])
                _ALL_VOWELS_SET.add(_ph_list[1])
        _initialized = True

    return sorted(_ALL_VOWELS_SET),phone_list


from typing import (
    Callable,
    Optional,
    Type,
    TypeVar,
)

E = TypeVar('E', bound=BaseException)
R = TypeVar('R')

def catch(f: Callable[..., R],
          *args,
          exception_cls_to_catch: Type[E] = Exception,
          **kwargs
          ) -> tuple[Optional[E], Optional[R]]:
    """
    Invoke the given callable. If the callable raises an instance of the
    specified exception class, return that exception, otherwise return the
    result of the callable.

    :param f: the callable to invoke

    :param args: positional arguments to the callable

    :param exception_cls_to_catch: class of exceptions to catch. The name is
           intentionally long in order to avoid collisions with the keyword
           arguments to the callable.

    :param kwargs: keyword arguments to the callable

    :return: Either a tuple of None and the return value of the callable or
             a tuple of the exception raised by the callable and None.

    >>> catch(int, '42')
    (None, 42)

    >>> catch(int, '42', base=16)
    (None, 66)

    >>> catch(int, '')
    (ValueError("invalid literal for int() with base 10: ''"), None)

    >>> catch(int, '', exception_cls_to_catch=ValueError)
    (ValueError("invalid literal for int() with base 10: ''"), None)

    >>> catch(int, '', exception_cls_to_catch=BaseException)
    (ValueError("invalid literal for int() with base 10: ''"), None)

    >>> catch(int, '', exception_cls_to_catch=NotImplementedError)
    Traceback (most recent call last):
    ...
    ValueError: invalid literal for int() with base 10: ''

    >>> catch(int, '', base=16)
    (ValueError("invalid literal for int() with base 16: ''"), None)

    >>> catch(int, '', base=16, exception_cls_to_catch=ValueError)
    (ValueError("invalid literal for int() with base 16: ''"), None)
    """
    try:
        return None, f(*args, **kwargs)
    except exception_cls_to_catch as e:
        return e, None


from datetime import datetime
import re

class Building(object):
    def __init__(self, name, addr, id, url):
        self.name = name
        self.addr = addr
        self.id = id
        self.url = url

        self.listings = []

    @property
    def num_listings(self):
        return len(self.listings)

    def add_listing(self, listing):
        self.listings.append(listing)
        listing.building = self

    def rows(self):
        return [l.serialize() for l in self.listings]

class Listing(object):
    def __init__(self, timestamp, id, unit, rent, beds, baths, sqft):
        self.id = id
        self.timestamp = float(timestamp)
        self.dttime = datetime.fromtimestamp(self.timestamp)
        self.unit = unit
        self.rent = rent

        if beds == 'studio':
            self.beds = 0
        else:
            self.beds = float(re.search(r'^([\d.]+)\s', beds).group(1))
        self.baths = float(re.search(r'^([\d.]+)\s', baths).group(1))
        self.sqft = int(re.search(r'^([\d,]+)\s', sqft).group(1).replace(',', ''))

        self.building = None

    def __repr__(self):
        attrs = (
            self.unit,
            f'{self.beds}B/{self.baths}BR',
            f'{self.sqft}sqft'
        )
        return '<Listing: ' + ', '.join(attrs) + '>'

    @property
    def year(self):
        return self.dttime.year

    @property
    def month(self):
        return self.dttime.month

    def serialize(self):
        d = {
            'year': self.year,
            'month': self.month,
            'id': self.id,
            'bldg': self.building.name,
            'unit': self.unit,
            'beds': self.beds,
            'baths': self.baths,
            'sqft': self.sqft,
            'rent': self.rent,
        }
        return d


# 对PM2.5值按年月两列进行统计分析,并使用分组柱状图可视化

import pandas as pd
import matplotlib.pyplot as plt

data_path = 'C:\\Users\\shine小小昱\\Desktop\\pm2.csv'
save_path = 'C:\\Users\\shine小小昱\\Desktop\\'

def collect_data():
    """
    数据获取
    :return:
    """
    f = open(data_path, encoding='utf-8')  # 中文文件名或中文路径必须这么打开
    data_df = pd.read_csv(f, header=0)  # 列名采用第一行
    return data_df

def inspect_data(data_df):
    """
    审查数据
    """
    print('数据预览：')
    print(data_df.head(10))

    print('数据基本信息：')
    print(data_df.info())

    print('数据内容统计：')
    print(data_df.describe())

def process_data(data_df):
    """
    数据处理
    """
    data_df.dropna(inplace=True)

    return data_df

def analyze_data(proc_df):
    """
    数据分析
    :return: 按年月多重分组后的df， 透视年列后的df
    """
    # 多重分组
    grpd_proc_df = proc_df.groupby(by=['Year', 'Month'])['PM'].mean()

    # pivot透视表                                     需横向透视列(不会变成列名)  透视列下需统计的值(无论个数必须是列表，否则不会变成列名)
    pvt_proc_df = pd.pivot_table(proc_df, index='Month', columns='Year', values=['PM'], aggfunc='mean')   # 层级列结构

    return grpd_proc_df, pvt_proc_df

def save_and_show(grpd_proc_df, pvt_proc_df):
    """
    结果展示与保存
    """
    grpd_proc_df.to_csv(save_path + '年月统计pm.csv', header=['mean pm'])
    pvt_proc_df.to_csv(save_path + '年透视图pm.csv')
    #  只有一个主列 可不指定列名
    pvt_proc_df['PM'].plot(kind='bar', title='PM2.5', rot=0)  # 数据本身只有一列值 但是因为有二层列 自动分组柱状图
    plt.ylabel('pm')
    plt.tight_layout()
    plt.savefig(save_path + 'pm分组柱状图.png')
    plt.show()

if __name__ == '__main__':
    all_data_df = collect_data()
    inspect_data(all_data_df)
    proc_data_df = process_data(all_data_df)
    grouped_proc_df, pivot_proc_df = analyze_data(proc_data_df)
    save_and_show(grouped_proc_df, pivot_proc_df)


""" Tools for calorimeter cluster manipulations

    Contributors:
        - Vitaly Vorobyev (vit.vorobiev@gmail.com)

    Created: July 2020
    Modified: August 2020
"""

from typing import NamedTuple

import jax.numpy as np
import jax

from .cartesian import Position, Momentum

dtype = np.ndarray

class Cluster(NamedTuple):
    """ Calorimeter cluster """
    energy: dtype  # energy
    costh: dtype  # cosine of the polar angle
    phi: dtype  # azimuth angle

    @staticmethod
    def from_ndarray(data: np.ndarray):
        """ """
        assert data.shape[1] == 3
        return Cluster(*[data[:, i] for i in range(3)])

    @property
    def as_array(self) -> (np.ndarray):
        return np.column_stack([self.energy, self.costh, self.phi])

    @property
    def sinth(self):
        """ Sine of the polar angle """
        return np.sqrt(1 - self.costh**2)

def cartesian_to_cluster(pos: Position, mom: Momentum, R=1000) -> (Cluster):
    """ Make cluster from Position and Momentum """
    # Find intersection with calorimeter (see the geometry sketch)
    pos_mom_scalar_product = pos.x*mom.px + pos.y*mom.py + pos.z*mom.pz
    mom_total = mom.ptot
    mom_total_squared = mom_total**2

    full_computation = False
    if full_computation:
        alpha = (-pos_mom_scalar_product + np.sqrt(
            pos_mom_scalar_product**2 + mom_total_squared * (R**2 - pos.r**2)
        )) / mom_total_squared
    else:  # takes into account that R >> particle flight length
        alpha = R / mom_total - pos_mom_scalar_product / mom_total_squared

    cluster_position = Position.from_ndarray(
        pos.as_array + alpha.reshape(-1, 1) * mom.as_array)

    return Cluster.from_ndarray(np.column_stack([
        mom_total, cluster_position.costh, cluster_position.phi
    ]))

def momentum_from_cluster(clu: Cluster) -> (Momentum):
    """ Assuming photon (massless particle) """
    sinth = clu.sinth
    return Momentum.from_ndarray(np.column_stack([
        clu.energy * sinth * np.sin(clu.phi),
        clu.energy * sinth * np.cos(clu.phi),
        clu.energy * clu.costh,
    ]))

momentum_from_cluster_jacobian = jax.vmap(jax.jacfwd(momentum_from_cluster))


# This Python file uses the following encoding: utf-8

from unittest import TestCase
from pandocfilters import Para, Str, Space, Header, BulletList, Span, Strong, Plain, Link, RawBlock

import json

import pandoc_numbering

from helper import init, createLink, createMetaList, createMetaMap, createMetaInlines, createListStr, createMetaString

def getMeta1():
    return {
        'pandoc-numbering': createMetaList([
            createMetaMap({
                'category': createMetaInlines(u'exercise'),
                'listing': createMetaInlines(u'Listings of exercises'),
                'sectioning': createMetaInlines(u'-.+.')
            })
        ])
    }

def getMeta2():
    return {
        'pandoc-numbering': createMetaList([
            createMetaMap({
                'category': createMetaInlines(u'exercise'),
                'listing': createMetaInlines(u'Listings of exercises'),
                'sectioning': createMetaInlines(u'-.+.'),
                'tab': createMetaString(u'2'),
                'space': createMetaString(u'4'),
            })
        ])
    }

def getMeta3():
    return {
        'pandoc-numbering': createMetaList([
            createMetaMap({
                'category': createMetaInlines(u'exercise'),
                'listing': createMetaInlines(u'Listings of exercises'),
                'sectioning': createMetaInlines(u'-.+.'),
                'tab': createMetaString(u'a'),
                'space': createMetaString(u'b'),
            })
        ])
    }

def test_listing_classic():
    init()

    meta = getMeta1()

    src = Para(createListStr(u'Exercise #'))
    pandoc_numbering.numbering(src['t'], src['c'], '', meta)
    src = Para(createListStr(u'Exercise (test) #'))
    pandoc_numbering.numbering(src['t'], src['c'], '', meta)

    doc = [[{'unMeta': meta}], []]
    pandoc_numbering.addListings(doc, '', meta)

    dest = [
        Header(1, ['', ['unnumbered'], []], createListStr(u'Listings of exercises')),
	    BulletList([
	    	[Plain([createLink(['', [], []], createListStr(u'0.0.1 Exercise'), ['#exercise:0.0.1', ''])])],
	    	[Plain([createLink(['', [], []], createListStr(u'0.0.2 test'), ['#exercise:0.0.2', ''])])]
	    ])
	]

    assert json.loads(json.dumps(doc[1])) == json.loads(json.dumps(dest))

def listing_latex(meta, color, tab, space):
    init()

    src = Para(createListStr(u'Exercise #'))
    pandoc_numbering.numbering(src['t'], src['c'], 'latex', meta)
    src = Para(createListStr(u'Exercise (test) #'))
    pandoc_numbering.numbering(src['t'], src['c'], 'latex', meta)

    doc = [[{'unMeta': meta}], []]
    pandoc_numbering.addListings(doc, 'latex', meta)

    dest = [
        Header(1, ['', ['unnumbered'], []], createListStr(u'Listings of exercises')),
        RawBlock(
            'tex',
            ''.join([
                '\\hypersetup{linkcolor=' + color + '}',
                '\\makeatletter',
                '\\newcommand*\\l@exercise{\\@dottedtocline{1}{' + tab + 'em}{' + space + 'em}}',
                '\\@starttoc{exercise}',
                '\\makeatother'
            ])
        )
    ]

    assert json.loads(json.dumps(doc[1])) == json.loads(json.dumps(dest))

def test_listing_latex():
    listing_latex(getMeta1(), 'black', '1.5', '3.3')

def test_listing_latex_color():
    meta = getMeta1()
    meta['toccolor'] = createMetaInlines(u'blue')
    listing_latex(meta, 'blue', '1.5', '3.3')

def test_listing_latex_tab_space():
	listing_latex(getMeta2(), 'black', '2.0', '4.0')

def test_listing_latex_tab_space_error():
	listing_latex(getMeta3(), 'black', '1.5', '3.3')


import os
import gzip
import shutil
import struct
import urllib

"""
The Code is moved from the Standford Tensorflow Course.
Some modifications are made for correction.
"""

os.environ['TF_CPP_MIN_LOG_LEVEL']='2' # to avoid the warning that CPU capability is not fully exploited

# from matplotlib import pyplot as plt
import numpy as np
import tensorflow as tf

from PIL import Image, ImageOps

def safe_mkdir(path):
    """ Create a directory if there isn't one already. """
    try:
        os.mkdir(path)
    except OSError:
        pass

def download_one_file(download_url,
                    local_dest,
                    expected_byte=None,
                    unzip_and_remove=False):
    """
    Download the file from download_url into local_dest
    if the file doesn't already exists.
    If expected_byte is provided, check if
    the downloaded file has the same number of bytes.
    If unzip_and_remove is True, unzip the file and remove the zip file
    """
    # To avoid the possible back-slash of '\' in the url which may lead to 404 Error
    download_url = download_url.replace('\\', '/')

    if os.path.exists(local_dest) or os.path.exists(local_dest[:-3]):
        print('%s already exists' %local_dest)
    else:
        print('Downloading %s' %download_url)
        local_file, _ = urllib.request.urlretrieve(download_url, local_dest)
        file_stat = os.stat(local_dest)
        if expected_byte:
            if file_stat.st_size == expected_byte:
                print('Successfully downloaded %s' %local_dest)
                if unzip_and_remove:
                    with gzip.open(local_dest, 'rb') as f_in, open(local_dest[:-3],'wb') as f_out:
                        shutil.copyfileobj(f_in, f_out)
                    os.remove(local_dest)
            else:
                print('The downloaded file has unexpected number of bytes')

def download_mnist(path):
    """
    Download and unzip the dataset mnist if it's not already downloaded
    Download from http://yann.lecun.com/exdb/mnist
    """
    safe_mkdir(path)
    url = 'http://yann.lecun.com/exdb/mnist'
    filenames = ['train-images-idx3-ubyte.gz',
                'train-labels-idx1-ubyte.gz',
                't10k-images-idx3-ubyte.gz',
                't10k-labels-idx1-ubyte.gz']
    expected_bytes = [9912422, 28881, 1648877, 4542]

    for filename, byte in zip(filenames, expected_bytes):
        download_url = os.path.join(url, filename)
        local_dest = os.path.join(path, filename)
        download_one_file(download_url, local_dest, byte, True)

def parse_data(path, dataset, flatten):
    if dataset != 'train' and dataset != 't10k':
        raise NameError('dataset must be train or t10k')

    label_file = os.path.join(path, dataset + '-labels-idx1-ubyte')
    with open(label_file, 'rb') as file:
        _, num = struct.unpack(">II", file.read(8))
        labels = np.fromfile(file, dtype=np.int8) #int8
        new_labels = np.zeros((num, 10))
        new_labels[np.arange(num), labels] = 1

    img_file = os.path.join(path, dataset + '-images-idx3-ubyte')
    with open(img_file, 'rb') as file:
        _, num, rows, cols = struct.unpack(">IIII", file.read(16))
        imgs = np.fromfile(file, dtype=np.uint8).reshape(num, rows, cols) #uint8
        imgs = imgs.astype(np.float32) / 255.0
        if flatten:
            imgs = imgs.reshape([num, -1])

    return imgs, new_labels

def read_mnist(path, flatten=True, num_train=55000):
    """
    Read in the mnist dataset, given that the data is stored in path
    Return three tuples of numpy arrays
    ((train_imgs, train_labels), (val_images, val_labels), (test_imgs, test_labels))
    """
    imgs, labels = parse_data(path, 'train', flatten)
    indices = np.random.permutation(labels.shape[0])
    train_idx, val_idx = indices[:num_train], indices[num_train:]
    train_img, train_labels = imgs[train_idx, :], labels[train_idx, :]
    val_img, val_labels = imgs[val_idx, :], labels[val_idx, :]
    test = parse_data(path, 't10k', flatten)
    return (train_img, train_labels), (val_img, val_labels), test

def get_resized_image(img_path, width, height, save=True):
    image = Image.open(img_path)
    # PIL is column major so you have to swap the places of width and height
    image = ImageOps.fit(image, (width, height), Image.ANTIALIAS)
    if save:
        image_dirs = img_path.split('/')
        image_dirs[-1] = 'resized_' + image_dirs[-1]
        out_path = '/'.join(image_dirs)
        if not os.path.exists(out_path):
            image.save(out_path)
    image = np.asarray(image, np.float32)
    return np.expand_dims(image, 0)

def generate_noise_image(content_image, width, height, noise_ratio=0.6):
    noise_image = np.random.uniform(-20, 20, (1, height, width, 3)).astype(np.float32)
    return noise_image * noise_ratio + content_image*(1-noise_ratio)


import argparse
import unittest
import json
import logging
import time

from ansible_integration_test import AnsibleIntegrationTest
from girder_client import HttpError

class EC2VolumeIntegrationTest(AnsibleIntegrationTest):

    def __init__(self, name, girder_url, girder_user, girder_password, aws_access_key_id,
                  aws_secret_access_key):
        super(EC2VolumeIntegrationTest, self).__init__(
            name, girder_url, girder_user, girder_password, aws_access_key_id,
            aws_secret_access_key)
        self._volume_id = None
        self._profile_id = None
        self._cluster_id = None

    def tearDown(self):
        super(EC2VolumeIntegrationTest, self).tearDown()
        if self._volume_id:
            # Try to detach to volume
            try:
                volume_url = 'volumes/%s/detach' % self._volume_id
                r = self._client.put(volume_url)

                volume_status_url = 'volumes/%s/status' % self._volume_id
                self._wait_for_status(volume_status_url, 'available', timeout=60)

            except HttpError as error:
                # if it doesn't exist continue
                if error.status != 400:
                    raise error

            # Try to delete the volume
            try:
                volume_url = 'volumes/%s' % self._volume_id
                r = self._client.delete(volume_url)
            except HttpError as error:
                # if it doesn't exist continue
                if error.status != 400:
                    raise error

    def create_volume(self):
        volume_url = 'volumes'
        body =  {
            "name": "testVolume",
            "profileId": self._profile_id,
            "size": 11,
            "type": "ebs",
            "zone": "us-west-2a"
        }

        r = self._client.post(volume_url, data=json.dumps(body))
        self._volume_id = r['_id']

    def attach_volume(self):
        attach_url = 'volumes/%s/clusters/%s/attach' % (self._volume_id, self._cluster_id)
        body = {
            'path': '/mnt/data/'
        }

        r = self._client.put(attach_url, data=json.dumps(body))

        volume_status_url = 'volumes/%s/status' % self._volume_id
        self._wait_for_status(volume_status_url, 'in-use', timeout=60)

    def detach_volume(self):
        volume_url = 'volumes/%s/detach' % self._volume_id
        r = self._client.put(volume_url)

        volume_status_url = 'volumes/%s/status' % self._volume_id
        self._wait_for_status(volume_status_url, 'available', timeout=60)

    def delete_volume(self):
        volume_url = 'volumes/%s' % self._volume_id
        r = self._client.delete(volume_url)

        self.assertEquals(r['status'], 'deleting')

        start = time.time();
        while True:
            time.sleep(1)
            try:
                r = self._client.get("/volumes/%s" % self._volume_id)
            except HttpError as error:
                if error.status == 400:
                    break
                else:
                    raise error

            if time.time() - start > 60:
                self.fail('Volume never moved out of status: %s' % r['ec2']['status'])

    def test(self):
        try:
            t0 = time.time()
            self.create_profile()
            logging.info("Finished creating profile (%s)" % (time.time() - t0)); t0 = time.time()
            self.create_cluster()
            logging.info("Finished creating cluster (%s)" % (time.time() - t0)); t0 = time.time()
            self.launch_cluster()
            logging.info("Finished launching cluster (%s)" % (time.time() - t0)); t0 = time.time()

            self.create_volume()
            logging.info("Finished creating volume (%s)" % (time.time() - t0)); t0 = time.time()

            self.attach_volume()
            logging.info("Finished attaching volume (%s)" % (time.time() - t0)); t0 = time.time()

            self.detach_volume()
            logging.info("Finished detaching volume (%s)" % (time.time() - t0)); t0 = time.time()

            self.delete_volume()
            logging.info("Finished deleting volume (%s)" % (time.time() - t0)); t0 = time.time()

            self.terminate_cluster()
            logging.info("Finished terminating cluster (%s)" % (time.time() - t0)); t0 = time.time()
        except HttpError as error:
            self.fail(error.responseText)
        except KeyboardInterrupt:
            self.fail("Interupted!")

if __name__ == '__main__':
    parser = argparse.ArgumentParser(parents=[AnsibleIntegrationTest.parser])
    parser.add_argument('-i', '--aws_access_key_id', help='', required=True)
    parser.add_argument('-a', '--aws_secret_access_key', help='', required=True)
    parser.add_argument('-l', '--loglevel', help='', required=False, default="INFO",
                        choices=["INFO", "WARNING", "ERROR", "DEBUG"])

    args = parser.parse_args()

    logging.basicConfig(level=(getattr(logging, args.loglevel)))
    logging.getLogger("requests").setLevel(logging.WARNING)

    suite = unittest.TestSuite()
    suite.addTest(EC2VolumeIntegrationTest('test', args.girder_url, args.girder_user,
        args.girder_password, args.aws_access_key_id, args.aws_secret_access_key))
    unittest.TextTestRunner().run(suite)


"""
Build and Jenkins environment handling

This file contains all the code that hardcodes details about the Jenkins build
agent environment, such as paths to various executables.
"""

import os

from common import ConfigurationError
from common import Compiler,System
import cmake
import agents
import re

# TODO: Check that the paths returned/used actually exists and raise nice
# errors instead of mysteriously breaking builds if the node configuration is
# not right.
# TODO: Clean up the different mechanisms used here; even for the ~same thing,
# different approaches may be used (some might set an environment variable,
# others use an absolute path, or set a CMake option).

def _to_version_tuple(version_string):
    return [int(x) for x in version_string.split('.')]

def _is_older_version(older, newer):
    return _to_version_tuple(older) < _to_version_tuple(newer)

class BuildEnvironment(object):
    """Provides access to the build environment.

    Most details of the build environment are handled transparently based on
    the provided build options, and the build script does not need to consider
    this.  For build scripts, the main interface this class provides is to find
    locations of some special executables (such as cppcheck) that may be needed
    for the build.  Compiler selection is handled without special action from
    build scripts.

    In rare cases, the build scripts may benefit from inspecting the attributes
    in this class to determine, e.g., the operating system running the build or
    the compiler being used.

    Attributes:
       system (System): Operating system of the build node.
       compiler (Compiler or None): Selected compiler.
       compiler_version (string): Version number for the selected compiler.
       c_compiler (str or None): Name of the C compiler executable.
       cxx_compiler (str or None): Name of the C++ compiler executable.
       libcxx_version (str or None): Version of libc++ to use
       doxygen_command (str): Name of the doxygen executable.
       gcov_command (str): Name of the gcov executable.
       cmake_command (str): Name of the CMake executable.
       ctest_command (str): Name of the CTest executable.
       cmake_version (str): Version of the CMake executable.
       cmake_generator (str or None): CMake generator being used.
       armhpc_version (str or None): The version of the ARM HPC toolchain.
       armpl_dir (str or None): the ARM Perf Libraries directory
       cuda_root (str or None): Root of the CUDA toolkit being used
           (for passing to CUDA_TOOLKIT_ROOT_DIR CMake option).
       libhwloc_root (str or None): Root of the hwloc library being used
           (for adding to CMake search path).
       opencl_version (str or None): OpenCL API version expected to be detected.
       cuda_host_compiler (str or None): Full path to the host compiler used
           with CUDA (for passing to CUDA_HOST_COMPILER CMake option).
       amdappsdk_root (str or None): Root of the AMD SDK being used
           (for using as AMDAPPSDKROOT environment variable).
       extra_cmake_options (Dict[str, str]): Additional options to pass to
           CMake.
    """

    def __init__(self, factory):
        self.system = factory.system
        self.compiler = None
        self.compiler_version = None
        self.c_compiler = None
        self.cxx_compiler = None
        self.doxygen_command = None
        self.gcov_command = None
        self.cmake_command = 'cmake'
        self.ctest_command = 'ctest'
        self.cmake_version = None
        self.cmake_generator = None
        self.armhpc_version = None
        self.armpl_dir = None
        self.cuda_root = None
        self.opencl_version = None
        self.cuda_host_compiler = None
        self.amdappsdk_root = None
        self.clang_analyzer_output_dir = None
        self.libcxx_version = None
        self.extra_cmake_options = dict()
        self.gcc_exe = None

        self._build_prefix_cmd = None
        self._cmd_runner = factory.cmd_runner
        self._workspace = factory.workspace
        self._node_name = factory.jenkins.node_name
        self._cmake_base_dir = None

        self._build_jobs = agents.get_default_build_parallelism(self._node_name)

        environment_command = agents.get_environment_subshell(self._node_name)
        if environment_command:
            environment_dump_command = environment_command + ' -- {0} -E environment'.format(self.cmake_command)
            self._cmd_runner.import_env(environment_dump_command)

        if self.system is not None:
            self._init_system()

    def get_cppcheck_command(self, version):
        """Returns path to the cppcheck executable of given version.

        Args:
            version (str): cppcheck version to use.
        """
        return os.path.expanduser('~/bin/cppcheck-{0}'.format(version))

    def get_doxygen_command(self, version):
        """Returns path to the Doxygen executable of given version.

        Args:
            version (str): Doxygen version to use.
        """
        return os.path.expanduser('~/tools/doxygen-{0}/bin/doxygen'.format(version))

    def get_uncrustify_command(self):
        """Returns path to the uncrustify executable."""
        return os.path.expanduser('~/bin/uncrustify')

    def get_clang_format_command(self, version):
        """Returns path to clang-format executable."""
        clang_format_name = 'clang-format-{0}'.format(version)
        return self._cmd_runner.find_executable(clang_format_name)

    def _get_build_cmd(self, target=None, parallel=True, keep_going=False):
        cmd = []
        if self._build_prefix_cmd is not None:
            cmd.extend(self._build_prefix_cmd)
        cmd.extend([self.cmake_command, '--build', '.'])
        if target is not None:
            cmd.extend(['--target', target])
        jobs = self._build_jobs if parallel else 1
        cmd.extend(['--', '-j{0}'.format(jobs)])
        if keep_going:
            cmd.append('-k')
        return cmd

    def _set_cmake_minimum_version(self, version):
        if self.cmake_version or not version:
            return
        current_version = cmake.get_cmake_version(self._cmd_runner, self.cmake_command)
        self.cmake_version = current_version
        if not _is_older_version(current_version, version):
            return
        available_versions = self._get_available_cmake_versions()
        for test_version in available_versions:
            if not _is_older_version(test_version, version):
                self._init_cmake(test_version)

    def _get_available_cmake_versions(self):
        versions = []
        for cmake_dir in os.listdir(self._cmake_base_dir):
            cmd = os.path.join(self._cmake_base_dir, cmake_dir, 'bin', 'cmake')
            if self.system == System.WINDOWS and not cmd.endswith('.exe'):
                cmd += '.exe'
            if os.path.isfile(cmd):
                versions.append(cmake_dir.split("-")[-1])
        versions.sort(key=_to_version_tuple)
        return versions

    def set_env_var(self, variable, value):
        """Sets environment variable to be used for further commands.

        All subsequent commands run with BuildContext.run_cmd() etc. will use
        the environment variable.

        Args:
            variable (str): Name of environment variable to set.
            value (str): Value to set the variable to.  As a convenience, if
                the value is None, nothing is done.
        """
        self._cmd_runner.set_env_var(variable, value)

    def append_to_env_var(self, variable, value):
        self._cmd_runner.append_to_env_var(variable, value)

    def prepend_path_env(self, path):
        """Prepends a path to the executable search path (PATH)."""
        self._cmd_runner.prepend_to_env_var('PATH', os.path.expanduser(path), sep=os.pathsep)

    def append_path_env(self, path):
        """Appends a path to the executable search path (PATH)."""
        self._cmd_runner.append_to_env_var('PATH', os.path.expanduser(path), sep=os.pathsep)

    def run_env_script(self, env_cmd):
        # Capture the environment created by sourcing env_cmd
        env_dump_cmd = env_cmd + ' && {0} -E environment'.format(self.cmake_command)
        self._cmd_runner.import_env(env_dump_cmd)

    def get_env_var(self, variable):
        return self._cmd_runner.get_env_var(variable)

    def _init_system(self):
        if self.system == System.WINDOWS:
            self.cmake_generator = 'NMake Makefiles JOM'
            self._cmake_base_dir = 'c:\\utils'
        else:
            self.prepend_path_env('~/bin')
            if self.system == System.OSX:
                self.set_env_var('CMAKE_PREFIX_PATH', '/opt/local')
            # If/when we want core dumps from builds, enable
            # the next line
            # self._init_core_dump()
            self._cmake_base_dir = ('/opt/cmake')

    def _init_core_dump(self):
        import resource
        try:
            limits = (resource.RLIM_INFINITY, resource.RLIM_INFINITY)
            resource.setrlimit(resource.RLIMIT_CORE, limits)
        except:
            pass

    # Methods from here down are used as build option handlers in options.py.
    # Please keep them in the same order as in process_build_options().

    def _set_build_jobs(self, jobs):
        self._build_jobs = jobs

    def _init_cmake(self, version):
        cmake_bin_dir = os.path.join(self._cmake_base_dir, 'cmake-' + version, 'bin')
        if not os.path.exists(cmake_bin_dir):
            cmake_bin_dir = os.path.join(self._cmake_base_dir, version, 'bin')
        self.cmake_command = os.path.join(cmake_bin_dir, 'cmake')
        self.ctest_command = os.path.join(cmake_bin_dir, 'ctest')
        self.cmake_version = version

    def _init_gcc(self, version):
        """Initializes the build to use given gcc version as the compiler.

        This method is called internally if the build options set the compiler
        (with gcc-X.Y), but it can also be called directly from a build script
        if the build does not use options.

        Args:
            version (str): GCC version number (major.minor) to use.
        """
        self.compiler = Compiler.GCC
        self.compiler_version = version
        self.c_compiler = 'gcc-' + version
        self.cxx_compiler = 'g++-' + version
        self.gcov_command = 'gcov-' + version

    # TODO relocate this function to just above _manage_stdlib declaration.
    def _locate_gcc(self, format_for_stdlib_flag, use_stdlib_through_env_vars):
        """Locates a gcc toolchain to use with this build

        Use this function to prepare compilers (gcc, icc or clang) to
        use the standard library from a particular gcc installation on
        the particular agent in use, since the default system gcc may
        be too old.

        Requires that self.compiler describes the actual compiler to use.

        Once the oldest supported version is GROMACS 2020, the
        format_for_stdlib_flag and use_stdlib_through_env_vars
        arguments will no longer be required.

        Args:
        format_for_stdlib_flag(str): A format string expecting a named
           gcctoolchain parameter containing the base part of the path
           to an installed gcc (e.g. '/usr/local' for '/usr/local/bin/gcc')
           to pass to the compiler so that it finds the matching libstdc++
           to use when compiling.
        use_stdlib_through_env_vars(bool): Whether to use CFLAGS/CXXFLAGS
           environment variables to set the C++ standard library for
           compilation, as used before GROMACS 2020

        """

        # Artefacts built by all C++ compilers require link-time
        # access to a C++ standard library, and often other libraries
        # such as for OpenMP or sanitizers are typically installed
        # alongside that standard library. So for gcc, we ensure that
        # we link to components from the matching gcc version, rather
        # than the system default gcc. For clang and icc, we link to a
        # gcc specified for each build agent.
        gcc_name=None
        if self.compiler == Compiler.GCC:
            gcc_name = self.c_compiler
        elif self.compiler == Compiler.CLANG or self.compiler == Compiler.INTEL:
            gcc_name = agents.get_default_gcc_for_libstdcxx(self._node_name)

        gcc_toolchain_path=None
        if gcc_name:
            gcc_exe = self._cmd_runner.find_executable(gcc_name)
            if self.compiler != Compiler.GCC:
                self.gcc_exe = gcc_exe
            gcc_exe_dirname = os.path.dirname(gcc_exe)
            gcc_toolchain_path = os.path.join(gcc_exe_dirname, '..')

        if gcc_toolchain_path:
            if format_for_stdlib_flag and use_stdlib_through_env_vars:
                stdlibflag=format_for_stdlib_flag.format(gcctoolchain=gcc_toolchain_path)
                self.append_to_env_var('CFLAGS', stdlibflag)
                self.append_to_env_var('CXXFLAGS', stdlibflag)
            # Linker flag needed for all (#2846)
            format_for_linker_flags="-Wl,-rpath,{gcctoolchain}/lib64 -L{gcctoolchain}/lib64"
            self.extra_cmake_options['CMAKE_CXX_LINK_FLAGS'] = format_for_linker_flags.format(gcctoolchain=gcc_toolchain_path)

    def _init_clang(self, version):
        """Initializes the build to use given clang version as the compiler.

        This method is called internally if the build options set the compiler
        (with clang-X.Y), but it can also be called directly from a build
        script if the build does not use options.

        Args:
            version (str): clang version number (major.minor) to use.
        """
        self.compiler = Compiler.CLANG
        self.compiler_version = version
        self.c_compiler = 'clang-' + version
        self.cxx_compiler = 'clang++-' + version
        # Symbolizer is only required for ASAN builds, but should not do any
        # harm to always set it (and that is much simpler).
        clang_path = self._cmd_runner.find_executable(self.c_compiler)
        clang_path = os.path.dirname(clang_path)
        symbolizer_path = os.path.join(clang_path, 'llvm-symbolizer')
        self.set_env_var('ASAN_SYMBOLIZER_PATH', symbolizer_path)
        # Test binaries compiled with clang OpenMP support need to
        # find at run time the libomp.so that matches the compiler
        # (libgomp.so is not suitable).
        self.set_env_var('LD_LIBRARY_PATH', os.path.join(clang_path, '../lib'))

    def _init_armclang(self, version):
        """Initializes the build to use given armclang version as the compiler.

        This method is called internally if the build options set the compiler
        (with armclang-X.Y), but it can also be called directly from a build
        script if the build does not use options.

        Args:
            version (str): armclang version number (major.minor) to use.
        """
        self.compiler = Compiler.ARMCLANG
        self.compiler_version = version

        self.c_compiler   = 'armclang'
        self.cxx_compiler = 'armclang++'

    def _init_libcxx(self, version):
        """Initializes the build to use given libc++ (from LLVM project)

        This method is called internally if the build options specify it
        (with libcxx-X.Y), but it can also be called directly from a build
        script if the build does not use options.

        This option is only valid with clang, and only valid with a clang
        version of the same number

        Args:
            version (str): libcxx version number (major.minor) to use.
        """
        self.libcxx_version = version

    def _init_icc(self, version):
        if self.system == System.WINDOWS:
            if self.compiler is None or self.compiler != Compiler.MSVC:
                raise ConfigurationError('need to specify msvc version for icc on Windows')
            self.c_compiler = 'icl'
            self.cxx_compiler = 'icl'
            self.compiler = Compiler.INTEL
            self.extra_cmake_options['CMAKE_EXE_LINKER_FLAGS'] = '"/machine:x64"'
            if version == '15.0':
                self.run_env_script(r'"C:\Program Files (x86)\Intel\Composer XE 2015\bin\compilervars.bat" intel64 vs' + self.compiler_version)
            # TODO remove the next clause when no matrices use it any more
            elif version == '16.0':
                self.run_env_script(r'"C:\Program Files (x86)\IntelSWTools\compilers_and_libraries_2016\windows\bin\compilervars.bat" intel64 vs' + self.compiler_version)
            elif re.match('^(\d\d)$', version):
                self.run_env_script(r'"C:\Program Files (x86)\IntelSWTools\compilers_and_libraries_20{0}\windows\bin\compilervars.bat" intel64 vs{1}'.format(version, self.compiler_version))
            else:
                raise ConfigurationError('invalid icc version: got icc-{0}. Try a version with two digits, e.g. 18 for 2018 release.'.format(version))
        else:
            self.c_compiler = 'icc'
            self.cxx_compiler = 'icpc'
            self.compiler = Compiler.INTEL
            if re.match('^(\d\d)$', version):
                self.run_env_script('. /opt/intel/compilers_and_libraries_20{0}/linux/bin/compilervars.sh intel64'.format(version))
            # TODO remove the next clause when no matrices use it any more
            elif version == '16.0':
                self.run_env_script('. /opt/intel/compilers_and_libraries_2016/linux/bin/compilervars.sh intel64')
            elif version == '15.0':
                self.run_env_script('. /opt/intel/composer_xe_2015/bin/compilervars.sh intel64')
            elif version == '14.0':
                self.run_env_script('. /opt/intel/composer_xe_2013_sp1/bin/compilervars.sh intel64')
            elif version == '13.0':
                self.run_env_script('. /opt/intel/composer_xe_2013/bin/compilervars.sh intel64')
            elif version == '12.1':
                self.run_env_script('. /opt/intel/composer_xe_2011_sp1/bin/compilervars.sh intel64')
            else:
                raise ConfigurationError('invalid icc version: got icc-{0}. Try a version with two digits, e.g. 18 for 2018 release.'.format(version))

        self.compiler_version = version

    def _init_msvc(self, version):
        self.compiler = Compiler.MSVC
        self.compiler_version = version
        if version == '2010':
            self.run_env_script(r'"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat" amd64')
        elif version == '2013':
            self.run_env_script(r'"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" amd64')
        elif version == '2015':
            self.run_env_script(r'"C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" amd64 -vcvars_ver=14.0')
        elif version == '2017':
            self.run_env_script(r'"C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" amd64')
        else:
            raise ConfigurationError('only Visual Studio 2010, 2013, 2015, and 2017 are supported, got msvc-' + version)
        self._cmd_runner.copy_env_var("caexcludepath", "INCLUDE") #fragile because function is case sensitive

    def _init_clang_static_analyzer(self, version):
        scan_build = 'scan-build-' + version
        cxx_analyzer = 'c++-analyzer-' + version
        html_output_dir = self._workspace.get_log_dir(category='scan_html')
        self.clang_analyzer_output_dir = html_output_dir
        self.set_env_var('CCC_CC', self.c_compiler)
        self.set_env_var('CCC_CXX', self.cxx_compiler)
        self.cxx_compiler = cxx_analyzer
        self._build_prefix_cmd = [scan_build,
                '-o', html_output_dir]

    def _init_doxygen(self, version):
        self.doxygen_command = os.path.expanduser('~/tools/doxygen-{0}/bin/doxygen'.format(version))

    def _init_armhpc(self, version):
        self.armhpc_version = version
        module_load = '. /usr/share/modules/init/sh && module load '
        if version == '18.3' or version == '18.4' or version == '19.3':
            self.run_env_script(module_load + 'Generic-AArch64/Ubuntu/16.04/arm-hpc-compiler/' + version)
        else:
            raise ConfigurationError('Only the ARM HPC compiler 18.3, 18.4, 19.3 are supported, was passed ' + version)

    def _init_armpl(self):
        if not self.armhpc_version:
            raise ConfigurationError('ARM Perf Libs are only supported in conjunction with the ARM HPC compilers' + version)
        else:
            # ARMPL module name has the form
            # Generic-AArch64/Ubuntu/16.04/gcc-X.Y.Z/armpl/J.K.L, where we assume Z == L == 0
            # Generic-AArch64/Ubuntu/16.04/arm-hpc-compiler-J.K/armpl/J.K.L, where L == 0
            module_load = '. /usr/share/modules/init/sh && module load '
            if self.compiler == Compiler.ARMCLANG:
                self.run_env_script(module_load + 'Generic-AArch64/Ubuntu/16.04/arm-hpc-compiler-' + self.compiler_version + '/armpl/' + self.armhpc_version + '.0')
            elif self.compiler == Compiler.GCC:
                self.run_env_script(module_load + 'Generic-AArch64/Ubuntu/16.04/gcc-' + self.compiler_version + '.0/armpl/' + self.armhpc_version + '.0')
            else:
                raise ConfigurationError('ARM Perf Libs will only work with armclang or gcc, but the compiler in use is: ' + self.cxx_compiler)
            self.armpl_dir = self.get_env_var('ARMPL_DIR')

    def _init_cuda(self, version):
        self.cuda_root = '/opt/cuda_' + version

    def _init_libhwloc(self, version):
        self.libhwloc_root = '/opt/hwloc/' + version
        self._cmd_runner.prepend_to_env_var('HWLOC_DIR', self.libhwloc_root)

    # TODO: remove when no longer supporting legacy matrix formats
    def _init_opencl_legacy(self, version):
        self.opencl_version = '1.1'

    def _init_opencl(self, version):
        self.opencl_version = version

    # TODO: remove this (and its docs) when no longer supporting legacy matrix formats
    def _init_amdappsdk(self, version):
        self.amdappsdk_root = '/opt/AMDAPPSDK-' + version

    def _init_clFFT(self, version):
        self.clFFT_root = '/opt/clFFT/' + version

    def _init_phi(self):
        self.extra_cmake_options['CMAKE_PREFIX_PATH'] = os.path.expanduser('~/utils/libxml2')

    def _init_atlas(self):
        self.set_env_var('CMAKE_LIBRARY_PATH', '/usr/lib/atlas-base')

    def _init_mpi(self):
        pass

    def _manage_stdlib(self, use_stdlib_through_env_vars):
        """Coordinates the C++ standard library to use in the build

        Artefacts built by all C++ compilers require link-time access
        to a C++ standard library. This works differently for each
        compiler, and can be influenced by build options or the
        operating system.

        Often the standard library comes from an installation of gcc,
        and it is generally necessary to avoid the main compiler
        finding the system default gcc, which may be too old to be
        useful. A particular gcc installation is pre-specified for
        each agent to cater for this.

        Requires that self.compiler and self.system are specified
        accurately.

        Args:
        use_stdlib_through_env_vars(bool): Whether to use CFLAGS/CXXFLAGS
           environment variables to set the C++ standard library for
           compilation, as used before GROMACS 2020

        """
        if self.compiler == Compiler.CLANG:
            # The clang compiler can use either libstdc++ or libc++.
            if self.libcxx_version is None:
                # Use libstdc++ from the pre-specified gcc for this
                # agent.
                self._locate_gcc('--gcc-toolchain={gcctoolchain}', use_stdlib_through_env_vars)
                return
            else:
                # Use libc++ from this clang installation
                if self.compiler_version != self.libcxx_version:
                    raise ConfigurationError('libcxx version must match clang version')
                self.append_to_env_var('CXXFLAGS', '-stdlib=libc++')
                return
        else:
            if self.libcxx_version is not None:
                raise ConfigurationError('libcxx only supported with clang')

        if self.compiler == Compiler.INTEL and self.system != System.WINDOWS:
            # The Intel compiler on Linux must use a libstdc++ from
            # the pre-specified gcc for this agent.
            self._locate_gcc('-gcc-name={gcctoolchain}/bin/gcc', use_stdlib_through_env_vars)
            return

        if self.compiler == Compiler.GCC:
            # Newer gcc needs to be linked against compatible standard
            # libraries from the pre-specified gcc for this agent.
            self._locate_gcc(None, use_stdlib_through_env_vars)
            return

        # The remaining compilers run on Windows and organize these
        # details themselves.

    def _finalize(self, use_stdlib_through_env_vars):
        """Manages handling that has complex dependencies between inputs

        Args:
        use_stdlib_through_env_vars(bool): Whether to use CFLAGS/CXXFLAGS
           environment variables to set the C++ standard library for
           compilation.
           Defaults to True, which is used by branches prior to GROMACS 2020.

        """
        if use_stdlib_through_env_vars is None:
            use_stdlib_through_env_vars = True
        self._manage_stdlib(use_stdlib_through_env_vars)


import os
from os import listdir
from os.path import isfile, join
import pandas as pd

from nltk.corpus import stopwords

SEPARADOR = ' '

def build_csv_with_data(working_folder, dataset_path, filename):
    all_folders = [element for element in listdir(dataset_path) if not isfile(join(dataset_path, element))]
    output_file_name = os.path.join(working_folder, filename)
    print(output_file_name)

    with open(output_file_name, 'w') as outputFile:
        for folder in all_folders:
            print(folder)
            folder_path = os.path.join(dataset_path, folder)
            all_files = [f for f in listdir(folder_path) if isfile(os.path.join(folder_path, f))]
            for file in all_files:
                text = ""
                with open(os.path.join(folder_path, file), encoding="utf8") as infile:
                    # firstLine = infile.readline().replace("\n","")
                    for line in infile:
                        # remove stopwords do texto de concatena as linhas para formar um CSV
                        filtered_words = [word for word in line.split(' ') if word not in stopwords.words('portuguese')]
                        line = SEPARADOR.join(filtered_words)
                        text = text + " " + line.replace("\n", "")
                outputFile.write(folder + "\t" + text + "\n")

    return output_file_name

def read_data(filepath):
    data_frame = pd.read_csv(filepath, encoding="ISO-8859-1", sep='\t', header=None, skiprows=0,
                             names=["Class", "Text"])

    return data_frame.groupby('Class').filter(lambda x: len(x) > 10)


# coding: utf-8

# In[1]:

# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All"
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session

# In[2]:

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'inline')
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score

# In[3]:

data = pd.read_csv('/kaggle/input/creditcardfraud/creditcard.csv')

# In[4]:

data.head()

# In[5]:

data.info()

# In[6]:

data['Class'].value_counts()

# In[9]:

# separating the data for analysis
legit = data[data.Class == 0]
fraud = data[data.Class == 1]

# In[10]:

legit_sample = legit.sample(n=492) # Handlimg imbalanced data

# In[11]:

new_dataset = pd.concat([legit_sample, fraud], axis=0)

# In[12]:

new_dataset['Class'].value_counts()

# In[13]:

X = new_dataset.drop(columns='Class', axis=1)
Y = new_dataset['Class']

# In[16]:

X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, stratify=Y, random_state=2)

# In[17]:

print(X.shape, X_train.shape, X_test.shape)

# In[18]:

model = LogisticRegression()

# In[20]:

# training the Logistic Regression Model with Training Data
model.fit(X_train, Y_train)

# In[21]:

# accuracy on training data
X_train_prediction = model.predict(X_train)
training_data_accuracy = accuracy_score(X_train_prediction, Y_train)

# In[22]:

print('Accuracy on Training data : ', training_data_accuracy)

# In[25]:

# accuracy on test data
X_test_prediction = model.predict(X_test)
test_data_accuracy = accuracy_score(X_test_prediction, Y_test)
print('Accuracy score on Test Data : ', test_data_accuracy)


"""
Tests for DBSCAN clustering algorithm
"""

import pickle
import warnings

import numpy as np
import pytest
from scipy.spatial import distance

from sklearn.cluster import DBSCAN, dbscan
from sklearn.cluster.tests.common import generate_clustered_data
from sklearn.metrics.pairwise import pairwise_distances
from sklearn.neighbors import NearestNeighbors
from sklearn.utils._testing import assert_array_equal
from sklearn.utils.fixes import CSR_CONTAINERS, LIL_CONTAINERS

n_clusters = 3
X = generate_clustered_data(n_clusters=n_clusters)

def test_dbscan_similarity():
    # Tests the DBSCAN algorithm with a similarity array.
    # Parameters chosen specifically for this task.
    eps = 0.15
    min_samples = 10
    # Compute similarities
    D = distance.squareform(distance.pdist(X))
    D /= np.max(D)
    # Compute DBSCAN
    core_samples, labels = dbscan(
        D, metric="precomputed", eps=eps, min_samples=min_samples
    )
    # number of clusters, ignoring noise if present
    n_clusters_1 = len(set(labels)) - (1 if -1 in labels else 0)

    assert n_clusters_1 == n_clusters

    db = DBSCAN(metric="precomputed", eps=eps, min_samples=min_samples)
    labels = db.fit(D).labels_

    n_clusters_2 = len(set(labels)) - int(-1 in labels)
    assert n_clusters_2 == n_clusters

def test_dbscan_feature():
    # Tests the DBSCAN algorithm with a feature vector array.
    # Parameters chosen specifically for this task.
    # Different eps to other test, because distance is not normalised.
    eps = 0.8
    min_samples = 10
    metric = "euclidean"
    # Compute DBSCAN
    # parameters chosen for task
    core_samples, labels = dbscan(X, metric=metric, eps=eps, min_samples=min_samples)

    # number of clusters, ignoring noise if present
    n_clusters_1 = len(set(labels)) - int(-1 in labels)
    assert n_clusters_1 == n_clusters

    db = DBSCAN(metric=metric, eps=eps, min_samples=min_samples)
    labels = db.fit(X).labels_

    n_clusters_2 = len(set(labels)) - int(-1 in labels)
    assert n_clusters_2 == n_clusters

@pytest.mark.parametrize("lil_container", LIL_CONTAINERS)
def test_dbscan_sparse(lil_container):
    core_sparse, labels_sparse = dbscan(lil_container(X), eps=0.8, min_samples=10)
    core_dense, labels_dense = dbscan(X, eps=0.8, min_samples=10)
    assert_array_equal(core_dense, core_sparse)
    assert_array_equal(labels_dense, labels_sparse)

@pytest.mark.parametrize("include_self", [False, True])
def test_dbscan_sparse_precomputed(include_self):
    D = pairwise_distances(X)
    nn = NearestNeighbors(radius=0.9).fit(X)
    X_ = X if include_self else None
    D_sparse = nn.radius_neighbors_graph(X=X_, mode="distance")
    # Ensure it is sparse not merely on diagonals:
    assert D_sparse.nnz < D.shape[0] * (D.shape[0] - 1)
    core_sparse, labels_sparse = dbscan(
        D_sparse, eps=0.8, min_samples=10, metric="precomputed"
    )
    core_dense, labels_dense = dbscan(D, eps=0.8, min_samples=10, metric="precomputed")
    assert_array_equal(core_dense, core_sparse)
    assert_array_equal(labels_dense, labels_sparse)

def test_dbscan_sparse_precomputed_different_eps():
    # test that precomputed neighbors graph is filtered if computed with
    # a radius larger than DBSCAN's eps.
    lower_eps = 0.2
    nn = NearestNeighbors(radius=lower_eps).fit(X)
    D_sparse = nn.radius_neighbors_graph(X, mode="distance")
    dbscan_lower = dbscan(D_sparse, eps=lower_eps, metric="precomputed")

    higher_eps = lower_eps + 0.7
    nn = NearestNeighbors(radius=higher_eps).fit(X)
    D_sparse = nn.radius_neighbors_graph(X, mode="distance")
    dbscan_higher = dbscan(D_sparse, eps=lower_eps, metric="precomputed")

    assert_array_equal(dbscan_lower[0], dbscan_higher[0])
    assert_array_equal(dbscan_lower[1], dbscan_higher[1])

@pytest.mark.parametrize("metric", ["precomputed", "minkowski"])
@pytest.mark.parametrize("csr_container", CSR_CONTAINERS + [None])
def test_dbscan_input_not_modified(metric, csr_container):
    # test that the input is not modified by dbscan
    X = np.random.RandomState(0).rand(10, 10)
    X = csr_container(X) if csr_container is not None else X
    X_copy = X.copy()
    dbscan(X, metric=metric)

    if csr_container is not None:
        assert_array_equal(X.toarray(), X_copy.toarray())
    else:
        assert_array_equal(X, X_copy)

@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
def test_dbscan_no_core_samples(csr_container):
    rng = np.random.RandomState(0)
    X = rng.rand(40, 10)
    X[X < 0.8] = 0

    for X_ in [X, csr_container(X)]:
        db = DBSCAN(min_samples=6).fit(X_)
        assert_array_equal(db.components_, np.empty((0, X_.shape[1])))
        assert_array_equal(db.labels_, -1)
        assert db.core_sample_indices_.shape == (0,)

def test_dbscan_callable():
    # Tests the DBSCAN algorithm with a callable metric.
    # Parameters chosen specifically for this task.
    # Different eps to other test, because distance is not normalised.
    eps = 0.8
    min_samples = 10
    # metric is the function reference, not the string key.
    metric = distance.euclidean
    # Compute DBSCAN
    # parameters chosen for task
    core_samples, labels = dbscan(
        X, metric=metric, eps=eps, min_samples=min_samples, algorithm="ball_tree"
    )

    # number of clusters, ignoring noise if present
    n_clusters_1 = len(set(labels)) - int(-1 in labels)
    assert n_clusters_1 == n_clusters

    db = DBSCAN(metric=metric, eps=eps, min_samples=min_samples, algorithm="ball_tree")
    labels = db.fit(X).labels_

    n_clusters_2 = len(set(labels)) - int(-1 in labels)
    assert n_clusters_2 == n_clusters

def test_dbscan_metric_params():
    # Tests that DBSCAN works with the metrics_params argument.
    eps = 0.8
    min_samples = 10
    p = 1

    # Compute DBSCAN with metric_params arg

    with warnings.catch_warnings(record=True) as warns:
        db = DBSCAN(
            metric="minkowski",
            metric_params={"p": p},
            eps=eps,
            p=None,
            min_samples=min_samples,
            algorithm="ball_tree",
        ).fit(X)
    assert not warns, warns[0].message
    core_sample_1, labels_1 = db.core_sample_indices_, db.labels_

    # Test that sample labels are the same as passing Minkowski 'p' directly
    db = DBSCAN(
        metric="minkowski", eps=eps, min_samples=min_samples, algorithm="ball_tree", p=p
    ).fit(X)
    core_sample_2, labels_2 = db.core_sample_indices_, db.labels_

    assert_array_equal(core_sample_1, core_sample_2)
    assert_array_equal(labels_1, labels_2)

    # Minkowski with p=1 should be equivalent to Manhattan distance
    db = DBSCAN(
        metric="manhattan", eps=eps, min_samples=min_samples, algorithm="ball_tree"
    ).fit(X)
    core_sample_3, labels_3 = db.core_sample_indices_, db.labels_

    assert_array_equal(core_sample_1, core_sample_3)
    assert_array_equal(labels_1, labels_3)

    with pytest.warns(
        SyntaxWarning,
        match=(
            "Parameter p is found in metric_params. "
            "The corresponding parameter from __init__ "
            "is ignored."
        ),
    ):
        # Test that checks p is ignored in favor of metric_params={'p': <val>}
        db = DBSCAN(
            metric="minkowski",
            metric_params={"p": p},
            eps=eps,
            p=p + 1,
            min_samples=min_samples,
            algorithm="ball_tree",
        ).fit(X)
        core_sample_4, labels_4 = db.core_sample_indices_, db.labels_

    assert_array_equal(core_sample_1, core_sample_4)
    assert_array_equal(labels_1, labels_4)

def test_dbscan_balltree():
    # Tests the DBSCAN algorithm with balltree for neighbor calculation.
    eps = 0.8
    min_samples = 10

    D = pairwise_distances(X)
    core_samples, labels = dbscan(
        D, metric="precomputed", eps=eps, min_samples=min_samples
    )

    # number of clusters, ignoring noise if present
    n_clusters_1 = len(set(labels)) - int(-1 in labels)
    assert n_clusters_1 == n_clusters

    db = DBSCAN(p=2.0, eps=eps, min_samples=min_samples, algorithm="ball_tree")
    labels = db.fit(X).labels_

    n_clusters_2 = len(set(labels)) - int(-1 in labels)
    assert n_clusters_2 == n_clusters

    db = DBSCAN(p=2.0, eps=eps, min_samples=min_samples, algorithm="kd_tree")
    labels = db.fit(X).labels_

    n_clusters_3 = len(set(labels)) - int(-1 in labels)
    assert n_clusters_3 == n_clusters

    db = DBSCAN(p=1.0, eps=eps, min_samples=min_samples, algorithm="ball_tree")
    labels = db.fit(X).labels_

    n_clusters_4 = len(set(labels)) - int(-1 in labels)
    assert n_clusters_4 == n_clusters

    db = DBSCAN(leaf_size=20, eps=eps, min_samples=min_samples, algorithm="ball_tree")
    labels = db.fit(X).labels_

    n_clusters_5 = len(set(labels)) - int(-1 in labels)
    assert n_clusters_5 == n_clusters

def test_input_validation():
    # DBSCAN.fit should accept a list of lists.
    X = [[1.0, 2.0], [3.0, 4.0]]
    DBSCAN().fit(X)  # must not raise exception

def test_pickle():
    obj = DBSCAN()
    s = pickle.dumps(obj)
    assert type(pickle.loads(s)) == obj.__class__

def test_boundaries():
    # ensure min_samples is inclusive of core point
    core, _ = dbscan([[0], [1]], eps=2, min_samples=2)
    assert 0 in core
    # ensure eps is inclusive of circumference
    core, _ = dbscan([[0], [1], [1]], eps=1, min_samples=2)
    assert 0 in core
    core, _ = dbscan([[0], [1], [1]], eps=0.99, min_samples=2)
    assert 0 not in core

def test_weighted_dbscan(global_random_seed):
    # ensure sample_weight is validated
    with pytest.raises(ValueError):
        dbscan([[0], [1]], sample_weight=[2])
    with pytest.raises(ValueError):
        dbscan([[0], [1]], sample_weight=[2, 3, 4])

    # ensure sample_weight has an effect
    assert_array_equal([], dbscan([[0], [1]], sample_weight=None, min_samples=6)[0])
    assert_array_equal([], dbscan([[0], [1]], sample_weight=[5, 5], min_samples=6)[0])
    assert_array_equal([0], dbscan([[0], [1]], sample_weight=[6, 5], min_samples=6)[0])
    assert_array_equal(
        [0, 1], dbscan([[0], [1]], sample_weight=[6, 6], min_samples=6)[0]
    )

    # points within eps of each other:
    assert_array_equal(
        [0, 1], dbscan([[0], [1]], eps=1.5, sample_weight=[5, 1], min_samples=6)[0]
    )
    # and effect of non-positive and non-integer sample_weight:
    assert_array_equal(
        [], dbscan([[0], [1]], sample_weight=[5, 0], eps=1.5, min_samples=6)[0]
    )
    assert_array_equal(
        [0, 1], dbscan([[0], [1]], sample_weight=[5.9, 0.1], eps=1.5, min_samples=6)[0]
    )
    assert_array_equal(
        [0, 1], dbscan([[0], [1]], sample_weight=[6, 0], eps=1.5, min_samples=6)[0]
    )
    assert_array_equal(
        [], dbscan([[0], [1]], sample_weight=[6, -1], eps=1.5, min_samples=6)[0]
    )

    # for non-negative sample_weight, cores should be identical to repetition
    rng = np.random.RandomState(global_random_seed)
    sample_weight = rng.randint(0, 5, X.shape[0])
    core1, label1 = dbscan(X, sample_weight=sample_weight)
    assert len(label1) == len(X)

    X_repeated = np.repeat(X, sample_weight, axis=0)
    core_repeated, label_repeated = dbscan(X_repeated)
    core_repeated_mask = np.zeros(X_repeated.shape[0], dtype=bool)
    core_repeated_mask[core_repeated] = True
    core_mask = np.zeros(X.shape[0], dtype=bool)
    core_mask[core1] = True
    assert_array_equal(np.repeat(core_mask, sample_weight), core_repeated_mask)

    # sample_weight should work with precomputed distance matrix
    D = pairwise_distances(X)
    core3, label3 = dbscan(D, sample_weight=sample_weight, metric="precomputed")
    assert_array_equal(core1, core3)
    assert_array_equal(label1, label3)

    # sample_weight should work with estimator
    est = DBSCAN().fit(X, sample_weight=sample_weight)
    core4 = est.core_sample_indices_
    label4 = est.labels_
    assert_array_equal(core1, core4)
    assert_array_equal(label1, label4)

    est = DBSCAN()
    label5 = est.fit_predict(X, sample_weight=sample_weight)
    core5 = est.core_sample_indices_
    assert_array_equal(core1, core5)
    assert_array_equal(label1, label5)
    assert_array_equal(label1, est.labels_)

@pytest.mark.parametrize("algorithm", ["brute", "kd_tree", "ball_tree"])
def test_dbscan_core_samples_toy(algorithm):
    X = [[0], [2], [3], [4], [6], [8], [10]]
    n_samples = len(X)

    # Degenerate case: every sample is a core sample, either with its own
    # cluster or including other close core samples.
    core_samples, labels = dbscan(X, algorithm=algorithm, eps=1, min_samples=1)
    assert_array_equal(core_samples, np.arange(n_samples))
    assert_array_equal(labels, [0, 1, 1, 1, 2, 3, 4])

    # With eps=1 and min_samples=2 only the 3 samples from the denser area
    # are core samples. All other points are isolated and considered noise.
    core_samples, labels = dbscan(X, algorithm=algorithm, eps=1, min_samples=2)
    assert_array_equal(core_samples, [1, 2, 3])
    assert_array_equal(labels, [-1, 0, 0, 0, -1, -1, -1])

    # Only the sample in the middle of the dense area is core. Its two
    # neighbors are edge samples. Remaining samples are noise.
    core_samples, labels = dbscan(X, algorithm=algorithm, eps=1, min_samples=3)
    assert_array_equal(core_samples, [2])
    assert_array_equal(labels, [-1, 0, 0, 0, -1, -1, -1])

    # It's no longer possible to extract core samples with eps=1:
    # everything is noise.
    core_samples, labels = dbscan(X, algorithm=algorithm, eps=1, min_samples=4)
    assert_array_equal(core_samples, [])
    assert_array_equal(labels, np.full(n_samples, -1.0))

def test_dbscan_precomputed_metric_with_degenerate_input_arrays():
    # see https://github.com/scikit-learn/scikit-learn/issues/4641 for
    # more details
    X = np.eye(10)
    labels = DBSCAN(eps=0.5, metric="precomputed").fit(X).labels_
    assert len(set(labels)) == 1

    X = np.zeros((10, 10))
    labels = DBSCAN(eps=0.5, metric="precomputed").fit(X).labels_
    assert len(set(labels)) == 1

@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
def test_dbscan_precomputed_metric_with_initial_rows_zero(csr_container):
    # sample matrix with initial two row all zero
    ar = np.array(
        [
            [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
            [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
            [0.0, 0.0, 0.0, 0.0, 0.1, 0.0, 0.0],
            [0.0, 0.0, 0.0, 0.0, 0.1, 0.0, 0.0],
            [0.0, 0.0, 0.1, 0.1, 0.0, 0.0, 0.3],
            [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.1],
            [0.0, 0.0, 0.0, 0.0, 0.3, 0.1, 0.0],
        ]
    )
    matrix = csr_container(ar)
    labels = DBSCAN(eps=0.2, metric="precomputed", min_samples=2).fit(matrix).labels_
    assert_array_equal(labels, [-1, -1, 0, 0, 0, 1, 1])


'''
Created on Dec, 201

@author: Will Yzaguirr
'''
from flask import Flask, flash, render_template, request, session
import os
import json
from app import app

app.secret_key = os.urandom(12)

current_user = ""

'''set global variable user_info with a dictionary of usernames and
passwords'''
with open('users.json', 'r') as jsonfile:
    user_data = json.loads(jsonfile.read())
users = [users['username'] for users in user_data]
passwords = [passw['password'] for passw in user_data]
user_info = dict(zip(users, passwords))

'''creates 2 global variables, songs, which is the available songs
to add to playlist, and all_songs which is all the songs. Used for
getting full song info'''
with open('songs.json', 'r') as jsonfile:
    song_data = json.loads(jsonfile.read())
songs = [(info['songname'], info['artist'], info['year'], info['album_cover']) for info in song_data]
all_songs = songs

'''create global variable user_playlist which is a dictionary with users
and their playlist in song title form'''
temp = []
user_playlist = []
playlists = [songs['playlist'] for songs in user_data]
for userP in playlists:
    for sng in userP:
        for titles in songs:
            if sng in titles:
                temp.append(titles)
    user_playlist.append(temp)
    temp = []

user_playlist = dict(zip(users, user_playlist))

@app.route('/')
def home():
    '''if you're not logged in, return the login page. Otherwise,
    return the playlist display page'''
    if not session.get('logged_in'):
        return render_template('login.html')
    else:
        global current_user
        order = []
        for i in range(0, len(user_playlist.get(current_user))):
            order.append(i + 1)
        return render_template("user_playlist.html",
                               playlist_length=len(user_playlist.get(current_user)),
                               current_user=current_user,
                               playlist=user_playlist.get(current_user),
                               all_songs=songs,
                               order = order)

@app.route('/login', methods=['POST'])
def do_admin_login():
    '''gets submitted username and password and checks if they match
    a user'''
    global current_user
    current_user = request.form['username']
    if request.form['username'] in user_info.keys() and request.form['password'] == user_info.get(request.form['username']):
        session['logged_in'] = True
    else:
        flash('incorrect login')
    return home()

@app.route("/logout")
def logout():
    '''logs out user'''
    session['logged_in'] = False
    return home()

@app.route("/submit")
def submit_changes():
    '''when submit is clicked, perform actions given in dropdown menu'''
    print("Songs:", songs)
    print("User play:", user_playlist)
    '''get dictionary change_log of song and action(add, remove)'''
    if request.method == "GET":
        current_user = request.args.get("user_info")

        if request.args.get("move") != "":
            start, end = request.args.get("move").split("_")[1] , request.args.get("move").split("_")[2]
            print(start, end)
            temp = user_playlist.get(current_user)[int(start) - 1]
            user_playlist.get(current_user)[int(start) - 1] = user_playlist.get(current_user)[int(end) - 1]
            user_playlist.get(current_user)[int(end) - 1] = temp

        change_log = {}
        for song in songs:
            if request.args.get(song[0].split(" ")[0]) != "nothing":
                change_log[song[0]] = request.args.get(song[0].split(" ")[0])

        print("CHange:", change_log)
        print("Current pl:", user_playlist.get(current_user))

        #to_del holds the place of the songs in user_playlist to be deleted
        to_del = []
        for song, action in change_log.items():
            if action == "remove":
                for i in range(len(user_playlist.get(current_user))):
                    print(i)
                    if song in user_playlist.get(current_user)[i]:
                        to_del.append(i)
                del user_playlist.get(current_user)[to_del[0]]
            else:
                temp = []
                for music in songs:
                    if song in music:
                        temp.append(music)
                user_playlist.get(current_user).append(temp[0])
        print("Changed play:", user_playlist.get(current_user))

        '''creates playlist of song titles to be written to json'''

        new_json_playlist = []
        for song_iter in user_playlist.get(current_user):
            new_json_playlist.append(song_iter[0])

        '''rewrites the json file with updated playlist info'''

        filename = 'users.json'
        with open(filename, 'r') as json_file:
            data = json.load(json_file)
        for users in data:
            if users['username'] == current_user:
                users['playlist'] = new_json_playlist

        os.remove(filename)
        with open(filename, 'w') as json_file:
            json.dump(data, json_file, indent=4)

        order = []
        for i in range(0, len(user_playlist.get(current_user))):
            order.append(i + 1)

        return render_template("user_playlist.html",
                               playlist_length = len(user_playlist.get(current_user)),
                               current_user = current_user,
                               playlist = user_playlist.get(current_user),
                               all_songs = songs,
                               order = order)

    else:
        order = []
        for i in range(0, len(user_playlist.get(current_user))):
            order.append(i + 1)
        return render_template("user_playlist.html",
                               playlist_length = len(user_playlist.get(current_user)),
                               current_user = current_user,
                               playlist = user_playlist.get(current_user),
                               all_songs = songs,
                               order = order)


import os
import warnings

import time
import tqdm
import json
import tempfile
import argparse

from torch.nn import Conv2d
from torch.nn.modules.conv import _ConvNd

from ...utils.runtime import get_instance
from ...utils.io import load, write_file

def enumerate_snapshots(path, ext='.gz'):
    path, _, snapshots = next(os.walk(os.path.abspath(path)))
    for basename, extension in map(os.path.splitext, snapshots):
        if extension != ext:
            continue

        # check file size
        snapshot = os.path.join(path, basename + ext)
        if os.stat(snapshot).st_size < 1:
            continue

        yield snapshot

def conv_filters(module, cls=Conv2d, prefix=''):
    """Collect filters from the convolutional layers of the specified model."""
    assert issubclass(cls, _ConvNd)

    for name, mod in module.named_modules(prefix=prefix):
        if not isinstance(mod, cls):
            continue

        yield name, mod.weight.cpu().detach().clone()

def main(root, tag='', force=False, debug=False):
    breakpoint() if debug else None

    primary_filename, *filenames = enumerate_snapshots(root)

    # the first snapshot becomes the primary model and is processed separately
    snapshot = load(primary_filename)
    master = snapshot['config']

    # the pack is created next to `root` under a similar name
    target = os.path.dirname(primary_filename)
    target = os.path.join(
        os.path.dirname(target),
        'kernels__' + os.path.basename(target)
        + ('__' if tag else '') + tag
    )
    os.makedirs(target, exist_ok=force)
    if force:
        # ensure that the directory being overwritten is empty.
        pass

    # load the model
    model = get_instance(**master['model'])
    model.load_state_dict(snapshot['state'], strict=True)

    # create tensor storage vault and commit the master model
    vault, sources = {}, [primary_filename]
    for name, weight in conv_filters(model):
        fid, vault[name] = tempfile.mkstemp(
            dir=target, prefix='v', suffix='.bin')
        write_file(weight, fid, save_size=False)
        os.close(fid)

    # check that all models have exactly the same config and commit them
    for filename in tqdm.tqdm(filenames, desc='fetching datasets'):
        snapshot = load(filename)
        if snapshot['config'] != master:
            # does not match the primary model
            warnings.warn(f'Config of `{filename}` does not match the'
                          f' the primary model `{primary_filename}`.',
                          RuntimeWarning)
            continue

        model.load_state_dict(snapshot['state'], strict=True)
        for name, weight in conv_filters(model):
            write_file(weight, open(vault[name], 'ab'), save_size=False)

        sources.append(filename)

    # make vault paths relative and compile meta info (from the last model)
    meta, filters = {}, list(conv_filters(model))
    common = os.path.commonpath([target, *vault.values()])
    for name, weight in tqdm.tqdm(filters, desc='saving datasets'):
        meta[name] = {
            'shape': (len(sources), *weight.shape),
            'dtype': repr(weight.dtype),
            'vault': os.path.normpath(vault[name].replace(common, '.'))
        }

    # the meta.json defines the parameters of the dataset
    with open(os.path.join(target, 'meta.json'), 'tw') as fout:
        json.dump({
            '__dttm__': time.strftime('%Y%m%d %H%M%S'),
            'dataset': meta,
            'config': master,
            'sources': sources,
        }, fout, indent=2)

parser = argparse.ArgumentParser(
    description='Preprocess model snapshots into kernel dataset, '
                'stored in an automatically generated directory.',
    add_help=True)

parser.add_argument(
    'root', type=str,
    help='The directory with stored model snapshots.')

parser.add_argument(
    '--tag', type=str, required=False,
    help='Optional suffix for the name of the dataset.')

parser.add_argument(
    '--force', required=False, dest='force', action='store_true',
    help='Overwrite experiment report if it already exists.')

parser.add_argument(
    '--debug', required=False, dest='debug', action='store_true',
    help='Enter trace mode.')

parser.set_defaults(tag='', force=False, debug=False)

main(**vars(parser.parse_args()))


import os
import nmap

# Define the subnet to scan
subnet = "192.168.0.0/24"

# Use nmap to scan the subnet for NFS shares
scanner = nmap.PortScanner()
scanner.scan(subnet, arguments="-p 2049")

# Loop through the hosts with open NFS ports
for host in scanner.all_hosts():
  # Check if the host has an open NFS port
  if scanner[host].has_tcp(2049):
    # Print the hostname and IP address of the NFS server
    print(f"Found NFS server: {host} ({scanner[host].hostname()})")

    # Get the list of NFS exports on the server
    exports = scanner[host]["tcp"][2049]["product"].split(",")

    # Loop through the exports
    for export in exports:
      # Mount the NFS export
      print(f"Mounting {export}")
      os.system(f"mount -t nfs {host}:{export} /mnt/{host}_{export.replace('/', '_')}")


import paddle
import paddle.nn as nn
import numpy as np

class PSNRLoss(nn.Layer):

    def __init__(self, loss_weight=1.0, reduction='mean'):
        super(PSNRLoss, self).__init__()
        assert reduction == 'mean'
        self.loss_weight = loss_weight
        self.scale = 10 / np.log(10)

    def forward(self, pred, target):
        assert len(pred.shape) == 4

        return self.loss_weight * self.scale * paddle.log(((pred - target) ** 2).mean(axis=(1, 2, 3)) + 1e-8).mean()

class ShuiyinLoss(nn.Layer):

    def __init__(self,
                 mode='l1', ):
        super(ShuiyinLoss, self).__init__()
        assert mode in ['l1', 'psnr', ]
        if mode == 'l1':
            self.main_loss = nn.L1Loss()
        if mode == 'psnr':
            self.main_loss = PSNRLoss()

    def forward(self, pred, gt):
        loss = self.main_loss(pred, gt)
        return loss


import os
import random
import re
from io import BytesIO
import PIL
from speak_and_listen import say, listen
import requests_html
from PIL import Image
# Quita el error por no verificar el certificado, linea 59 (funcion mostrar imagen)
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

# Global variables
p1_Wins = 0
p2_Wins = 0
rounds_played = 0

def get_categories(session):
    site_home = session.get("https://www.coolmod.com/")
    return site_home.html.find(".subfamilyheadertittle")

def let_user_choose_category():
    category_dict = {"coolpc": 'https://www.coolmod.com/pcs-coolmod/',
                     "portátiles": 'https://www.coolmod.com/pcs-tablets-portatiles-portatiles/',
                     "PCs sobremesa": 'https://www.coolmod.com/pcs-tablets-portatiles-pcs-sobremesa/',
                     "componentes PC": 'https://www.coolmod.com/componentes-hardware-componentes-pc/'}

    say("Para esta primera ronda, puedes elegir categoría.")

    keys = []
    values = []
    for key, value in category_dict.items():
        keys.append(key)
        values.append(value)

    chosen_category_link = None
    while not chosen_category_link:
        os.system("cls")
        print("Categorías a elegir:\n" + ("\n".join(keys)) + "\n")  # Pendiente de arreglar
        user_input = listen()

        if user_input in keys:
            chosen_category = keys[keys.index(user_input)]
            say("Has elegido la categoría " + chosen_category)
            chosen_category_link = category_dict[chosen_category]
            os.system("cls")
            return chosen_category_link
        else:
            say("Lo siento, no te he entendido, repite tu elección por favor.")

def get_random_category_link(all_categories):
    category = random.choice(all_categories)

    while category.text == "Configura tu PC a Medida":
        category = random.choice(all_categories)
        # Ha tocado el configurador
    return list(category.links)[0]

def proccess_products(session, category_link):
    product_category_page = session.get(category_link)
    products = product_category_page.html.find(".productInfo")

    product = random.choice(products)

    # Get product name
    product_name = product.text.split("\n")[1]

    # Get product price
    product_price = product.text.split("\n")[-1]
    product_price = float(product_price.replace("€", "").replace(" ", "").replace(".", "").replace(",", "."))
    # Quitar el sinbolo € y los espacios. Arreglar el numero para convertirlo en float

    # Get product image
    img_src = product.html.split("\n")[5]
    product_image_link = re.findall("(https://[A-aZ-z./-9-0]+)", img_src)[0]

    return product_name, product_price, product_image_link

def listen_and_get_user_guess():
    user_guess = listen()
    try:
        user_guess = user_guess.replace(" €", "").replace("euros", "")\
            .replace(",", ".").replace(" con ", ".").replace(" coma ", ".")
        final_user_guess = float(user_guess)
        return final_user_guess
    except ValueError:
        say("Lo siento, no te he entendido. ¿Te importaría repetirlo?")
        return listen_and_get_user_guess()
    except AttributeError:
        say("Lo siento, no te he entendido. ¿Te importaría repetirlo?")
        return listen_and_get_user_guess()

def show_product_image(product_image_link, session):
    try:
        product_image_downloaded = session.get(product_image_link, verify=False)
        image = Image.open(BytesIO(product_image_downloaded.content))
        image.show()
    except PIL.UnidentifiedImageError:
        print("No se ha podido mostrar la imagen")
    pass

def ask_and_check_user_guess(product_name, product_image_link, session):
    print("Nombre del producto {}".format(product_name))
    show_product_image(product_image_link, session)
    say("El nombre del producto es {}".format(product_name))

    say("¿Cuanto vale? Jugador 1")
    p1_guess = listen_and_get_user_guess()

    say("¿Cuanto vale? Jugador 2")
    p2_guess = listen_and_get_user_guess()

    return p1_guess, p2_guess

def set_round_number():
    print("Cuantas rondas te gustaría jugar? Debe de ser un numero impar.")
    say("Cuantas rondas te gustaría jugar? Debe de ser un numero impar")
    numbers_in_words = ["cero", "uno", "dos", "tres", "cuatro", "cinco", "seis", "siete", "ocho", "nueve"]

    user_input = listen()
    try:
        user_input = user_input.split(" ")[0]  # In case input ~= 5 rondas
    except AttributeError:
        say("Lo siento, no te he entendido")
        return set_round_number()

    if user_input in numbers_in_words:
        user_input = numbers_in_words.index(user_input)
    elif user_input == "una":
        user_input = 1

    try:
        number_of_rounds = int(user_input)
    except (ValueError, AttributeError, TypeError):
        say("Lo siento, no te he entendido")
        return set_round_number()

    if (number_of_rounds % 2) == 0:  # El numero es par
        say(str(number_of_rounds) + " no es un numero impar")
        return set_round_number()

    print("La partida durará {} rondas.".format(number_of_rounds))
    say("La partida durará {} rondas.".format(number_of_rounds))
    return number_of_rounds

def pick_random_winner():
    global p1_Wins
    global p2_Wins

    if random.randint(1, 2) == 1:
        p1_Wins += 1
        print_round_winner("jugador 1")
        say("La ronda la gana el jugador 1.")
    else:
        p2_Wins += 1
        print_round_winner("jugador 2")
        say("La ronda la gana el jugador 2.")
    return

def determine_winner(p1_guess, p2_guess, product_price):
    global p1_Wins
    global p2_Wins
    global rounds_played

    p1_absolute_error = round(abs(p1_guess - product_price), 2)
    p2_absolute_error = round(abs(p2_guess - product_price), 2)

    if p1_absolute_error < p2_absolute_error:
        p1_Wins += 1
        os.system("cls")
        print_round_winner("jugador 1")
        say("El primer jugador se ha acercado más. Se ha quedado a {} euros".format(p1_absolute_error))
    elif p1_absolute_error > p2_absolute_error:
        p2_Wins += 1
        os.system("cls")
        print_round_winner("jugador 2")
        say("El segundo jugador se ha acercado más. Se ha quedado a {} euros".format(p2_absolute_error))
    else:  # If both errors are the same
        os.system("cls")
        say("Ambos jugadores se han quedado igual de cerca. Se elegirá un ganador aleatorio.")
        pick_random_winner()
    rounds_played += 1
    return

def print_title():
    title = "EL PRECIO JUSTO by Unai Iglesias"
    print("+" + "-" * len(title) + "+")
    print("|" + title + "|")
    print("+" + "-" * len(title) + "+")
    return

def print_round_counter_and_scoreboard(number_of_rounds):
    global p1_Wins
    global p2_Wins
    global rounds_played
    content = "Ronda " + str(rounds_played + 1) + " / " + str(number_of_rounds)
    scoreboard = "J1 {} --- {} J2".format(p1_Wins, p2_Wins)
    print("+" + "-" * len(content) + "+")
    print("|" + content + "|" + "    " + scoreboard)
    print("+" + "-" * len(content) + "+")
    return

def print_round_winner(round_winner):
    content = "El " + round_winner + " ha ganado esta ronda"
    scoreboard = "J1 {} --- {} J2".format(p1_Wins, p2_Wins)
    print("+" + "-" * len(content) + "+")
    print("|" + content + "|" + "    " + scoreboard)
    print("+" + "-" * len(content) + "+")
    return

def print_winner(winner):
    content = "Ha ganado el " + winner + " ¡Felicidades!"
    scoreboard = "J1 {} --- {} J2".format(p1_Wins, p2_Wins)
    print("+" + "-" * len(content) + "+")
    print("|" + content + "|" + "    " + scoreboard)
    print("+" + "-" * len(content) + "+")
    return

def main():
    # Initialization
    global p1_Wins
    global p2_Wins
    global rounds_played
    os.system("cls")
    session = requests_html.HTMLSession()
    winner = None
    print_title()
    say("Bienvenidos a El precio justo, vamos a adivinar el precio de algunos productos")

    all_categories = get_categories(session)

    number_of_rounds = set_round_number()

    # End of initialization

    # Main game loop
    while rounds_played < number_of_rounds:
        os.system("cls")

        if rounds_played == 0:  # User will choose category for the first round
            category_link = let_user_choose_category()
        else:
            category_link = get_random_category_link(all_categories)

        print_round_counter_and_scoreboard(number_of_rounds)

        product_name, product_price, product_image_link = proccess_products(session, category_link)

        # Ask for guesses
        p1_guess, p2_guess = ask_and_check_user_guess(product_name, product_image_link, session)

        # Reveal real product price
        product_price_adapted_to_speech = str(round(product_price, 2)).replace(".", "coma")
        say("El precio era {} euros".format(product_price_adapted_to_speech))
        print("El precio era {} €.".format(round(product_price, 2)))

        determine_winner(p1_guess, p2_guess, product_price)

        if rounds_played != number_of_rounds:
            say("Siguiente ronda.")

    if p1_Wins > p2_Wins:
        winner = "primer jugador"
        os.system("cls")
    elif p2_Wins > p1_Wins:
        winner = "segundo jugador"
        os.system("cls")

    print_winner(winner)

    say("Ha ganado el {}. ¡Felicidades!".format(winner))
    os.system("pause")

if __name__ == "__main__":
    main()