import os, io, sys, json, pickle, re, mimetypes, shutil
from scripts.util.thread_events import check_stop_script
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from googleapiclient.http import MediaFileUpload, MediaIoBaseDownload

# Value to represent no part
NO_PART = "NO_PART"

# Custom Exception for when credentials fail to load
class CredentialsError(Exception):
    pass

# Returns whether or not the script is running in an exe/frozen file
def is_frozen():
    return getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS')

# Builds the drive service
def build_service():
    check_stop_script()
    return build('drive', 'v3', credentials=fetch_credentials())

# Fetches the user's credentials, assume credential files are in root directory
def fetch_credentials():
    """Loads the user's credentials from credentials.json or a token
    """
    # If modifying these scopes, delete the file token.pickle.
    SCOPES = ['https://www.googleapis.com/auth/drive']

    creds = None
    # The file token.pickle stores the user's access and refresh tokens, and is
    # created automatically when the authorization flow completes for the first
    # time.
    res_paths = parse_options('res_paths.json')
    token_path = resourcePath(os.path.join(res_paths['res-path'], 'res/token.pickle'))
    if os.path.exists(token_path):
        with open(token_path, 'rb') as token:
            creds = pickle.load(token)
    # If there are no (valid) credentials available, let the user log in.
    if not creds or not creds.valid:
        if creds and creds.expired and creds.refresh_token:
            creds.refresh(Request())
        else:
            creds_path = res_paths['creds-path']
            try:
                flow = InstalledAppFlow.from_client_secrets_file(
                    resourcePath(creds_path), SCOPES)
                creds = flow.run_local_server(port=0)
            except (UnicodeDecodeError, ValueError, OSError):
                res_paths['creds-path'] = ''
                write_options(res_paths, 'res_paths.json')
                raise CredentialsError(f'ERROR: Credentials file "{os.path.basename(creds_path)}" invalid. Please restart the application.')

        # Save the credentials for the next run
        with open(resourcePath(token_path), 'wb') as token:
            pickle.dump(creds, token)
    return creds

# Path to resources
def resourcePath(rel_path):
    """ Get absolute path to resource, works for dev and for PyInstaller """
    if is_frozen():
        return os.path.join(getattr(sys, '_MEIPASS', ''), rel_path)
    return rel_path

# Make Application Data files
def make_application_data(data_dir, recopy=False):
    path_exists = os.path.exists(data_dir)
    if path_exists and not recopy: return

    if not path_exists:
        os.makedirs(data_dir)

    src = resourcePath("./res")
    shutil.copytree(src, os.path.join(data_dir, 'res'), dirs_exist_ok=True)

    # Remove token.pickle file
    pickle = os.path.join(data_dir, 'res/token.pickle')
    if os.path.isfile(pickle):
        os.remove(pickle)

# Gets the path to the specified location within the resources directory
def get_full_path(partial_path):
    with open(resourcePath('res/options/res_paths.json')) as f:
        res_path = json.load(f)['res-path']

    if (res_path == '' or res_path == '.'):
        return resourcePath(partial_path)
    elif is_frozen():
        return os.path.join(res_path, partial_path)
    else:
        return partial_path

# Write to options json file
def write_options(options, filename, path = "res/options/"):
    full_path = os.path.join(get_full_path(path), filename)

    try:
        with open(full_path, 'w') as f:
            json.dump(options, f, indent=4)
    except OSError:
        print(f'ERROR: could not write to file "{full_path}"')

# Parse json options file
def parse_options(filename, path = "res/options/", from_=None):
    if not from_ or not os.path.isdir(from_):
        full_path = os.path.join(get_full_path(path), filename)
    else:
        full_path = os.path.join(from_, path, filename)

    try:
        with open(full_path) as f:
            return json.load(f)
    except OSError:
        print(f'ERROR: could not parse file "{full_path}"')
        return None

# Parse title of a file into [chartname, partname (if applicable), mimeType]
def parse_file(filename, alias_map=None):
    # title - part format
    match = re.search('(.*) - (.*)\.(.*)', filename)
    if match:
        return match.group(1), (alias_map.get(match.group(2)) or NO_PART) if alias_map else None, mimetypes.guess_type(filename)[0]

    # other file type format
    match = re.search('(.*)\.[\w+]', filename)
    if match:
        return match.group(1), None, mimetypes.guess_type(filename)[0]
    return None, None, None

# Convert dict of Part Name : [Valid Asiases] to dict of Alias : Part Name
def make_alias_map(parts_dict):
    alias_map = {}
    for part in parts_dict.keys():
        for alias in parts_dict[part]:
            alias_map[alias] = part
    return alias_map

# Gets the desired chart folder, searching in the directories in the id list. Returns a dictionary containing chart id and parent id
def get_chart_id(service, chart, id_list):
    for id in id_list:
        results = service.files().list(corpora="user",
                                       pageSize=1000,
                                       fields="files(id)",
                                       q=f'mimeType = "application/vnd.google-apps.folder" and'
                                         f'"{id}" in parents and '
                                         f'name="{chart}"',
                                       spaces="drive").execute()
        items = results.get('files', [])
        if len(items) == 1:
            return { "chart_id": items[0].get('id'), "parent_id": id }

    print(f'ERROR: "{chart}" folder not found in Digital Library directory')
    return { "chart_id": None, "parent_id": None }

# Gets the chart's parts folder, return its id (or None)
def get_parts_and_audio_folders(service, chart, chart_id):
    output = []
    for folder_name in ("Parts", "Audio"):
        # Search for parts folder
        folder_results = service.files().list(corpora="user",
                                            fields="files(id)",
                                            pageSize=1000,
                                            q=f'mimeType = "application/vnd.google-apps.folder" and'
                                            f'"{chart_id}" in parents and'
                                            f'name="{folder_name}"',
                                            spaces="drive").execute()
        items = folder_results.get('files', [])
        if len(items) != 1:
            print(f"ERROR: {folder_name} folder not found for current chart with name:", chart)
            output.append(None)
        else:
            output.append(items[0].get('id'))

    return tuple(output)

# Creates a folder
def make_folder(service, name, parent, makePublic=False):

    file_metadata = {
        'name': name,
        'mimeType': 'application/vnd.google-apps.folder',
        'parents': [parent]
    }
    try:
        response = service.files().create(body=file_metadata, fields='id').execute()
        folderId = response.get('id')
        if makePublic:
            service.permissions().create(fileId=folderId, body={'type': 'anyone', 'role': 'reader'})
        return response.get('id')
    except HttpError:
        print(f'ERROR: Unable to create folder "{name}"')
        return None

# Moves a file (or folder)
def move_file(service, file_id, old_parent, new_parent):
    try:
        service.files().update(fileId=file_id,
                                addParents=new_parent,
                                removeParents=old_parent,
                                fields='id, parents').execute()
    except HttpError:
        print(f'ERROR: Unable to move file')

# Creates a shortcut
def make_shortcut(service, name, target_id, parent):
    shortcut_metadata = {
        'name': name,
        'mimeType': 'application/vnd.google-apps.shortcut',
        'shortcutDetails': {
            'targetId': target_id
        },
        'parents': [parent]
    }
    service.files().create(body=shortcut_metadata).execute()

# Update an existing file's metadata and content.
# Based on: https://developers.google.com/drive/api/v2/reference/files/update
# Modified to work with v3 api
def update_file(service, file_id, new_filename, new_title=None, new_description=None, new_mime_type=None):
    try:
        # First retrieve the file from the API.
        file = service.files().get(fileId=file_id).execute()

        # File's new metadata.
        del file["id"]
        if (new_title): file['title'] = new_title
        if (new_description): file['description'] = new_description
        if (new_mime_type): file['mimeType'] = new_mime_type

        # File's new content.
        media_body = MediaFileUpload(new_filename, mimetype=new_mime_type, resumable=True)

        # Send the request to the API.
        updated_file = service.files().update(
            fileId=file_id,
            body=file,
            media_body=media_body,
            fields='id').execute()
        return updated_file["id"]
    except HttpError:
        print(f'Error when attempting to update file: {new_filename}')
        return None

# Upload a file to a folder in the drive
def upload_file(service, new_filename, display_name, parent, title=None, mime_type=None):
    try:
        # Create file metadata
        file_metadata = {
            'name' : display_name,
            'parents': [parent]
        }
        if title: file_metadata['title'] = title
        if mime_type: file_metadata['mimeType'] = mime_type

        # Make the file
        media = MediaFileUpload(new_filename, mimetype=mime_type)
        return service.files().create(body=file_metadata, media_body=media, fields='id').execute().get('id')
    except HttpError:
        print(f'WARNING: Unable to create file "{display_name}"')
        return None

# Downloads a file (or folder) from the drive
def download_file(service, file_id, dir, file_name, verbose=False):
    try:
        # Get target file
        request = service.files().get_media(fileId=file_id)

        # Resolve destination
        if not os.path.exists(dir): os.makedirs(dir)
        fh = io.FileIO(os.path.join(dir, file_name), "w")

        # Execute download
        downloader = MediaIoBaseDownload(fh, request)
        done = False
        if verbose: print(f'Starting download of "{file_name}"')
        while done is False:
            status, done = downloader.next_chunk()
            if verbose: print(f'Progress: {status.progress() * 100}%')
        if verbose: print("Completed download")
    except (OSError, HttpError):
        print(f'WARNING: Unable to download "{file_name}"')

# Returns the id(s) of a folder with the given id or name (optionally can specify parent directory)
def get_folder_ids(service, id = None, name = None, parent = None):
    # Check folder by id
    if (id != None):
        res = service.files().get(fileId=id, fields="id").execute()
        if res.get("id") and res["id"] == id: return [id]
        return None

    if (name == None): return None

    # Check folder by name + parent (optional)
    qstring = f'name="{name}" and mimeType="application/vnd.google-apps.folder"'
    if (parent != None): qstring += f' and "{parent}" in parents'

    results = service.files().list(q=qstring,
                                    corpora='user',
                                    pageSize=1000,
                                    fields='files(id)',
                                    includeItemsFromAllDrives=True,
                                    supportsAllDrives=True).execute()
    return [ res.get("id") for res in results.get("files") ] if results.get("files") else None

# Verify and return the digital library
def get_digital_library(service):
    """Verify that the DigitalLibrary folder and subfolders are in the correct locations.
    """
    check_stop_script()
    library_id, full_dig_id, current_id, future_id, archive_id = None, None, None, None, None

    # DigitalLibrary folder
    library_res = get_folder_ids(service, name="DigitalLibrary", parent="root")
    if not has_unique_folder(library_res, "DigitalLibrary", "My Drive"): return {}
    library_id = library_res[0]

    # Digitized Chart Data folder
    full_dig_res = get_folder_ids(service, name="LSJUMB Full Digitized Chart Data", parent=library_id)
    if not has_unique_folder(full_dig_res, "LSJUMB Full Digitized Chart Data", "DigitalLibrary"): return {}
    full_dig_id = full_dig_res[0]

    # Current Chartz folder
    current_res = get_folder_ids(service, name="Current Chartz", parent=full_dig_id)
    if not has_unique_folder(current_res, "Current Chartz", "LSJUMB Full Digitized Chart Data"): return {}
    current_id = current_res[0]

    # Old Chartz folder
    past_res = get_folder_ids(service, name="Old Chartz", parent=full_dig_id)
    if not has_unique_folder(past_res, "Old Chartz", "LSJUMB Full Digitized Chart Data"): return {}
    past_id = past_res[0]

    # Future Chartz folder
    future_res = get_folder_ids(service, name="Future Chartz", parent=full_dig_id)
    if not has_unique_folder(future_res, "Future Chartz", "LSJUMB Full Digitized Chart Data"): return {}
    future_id = future_res[0]

    # Archived Chartz folder
    archive_res = get_folder_ids(service, name="Archived Chartz", parent=full_dig_id)
    if not has_unique_folder(archive_res, "Archived Chartz", "LSJUMB Full Digitized Chart Data"): return {}
    archive_id = archive_res[0]

    return {"library_id": library_id, "current_id": current_id, "past_id": past_id, "future_id": future_id, "archive_id": archive_id}

# Get seperated section or separated sibelius parts folders
def get_separated_folders(service, library_id):
    check_stop_script()
    separated_ids = {}
    folder_names = [("Separated Sibelius Files", "sib"), ("Separated Section Parts", "sec"), ("Separated Part Audio", "aud")]
    for folder_name, abbr in folder_names:
        # Look for folder within Digital Library
        ids = get_folder_ids(service, name=folder_name, parent=library_id)
        if not has_unique_folder(ids, folder_name, "Digital Library"): return None
        separated_ids[abbr] = ids[0]

        # Look for Current, Old, Future, Archived Chartz folders
        for age_name, age in [("Current Chartz", "curr"), ("Old Chartz", "old"), ("Future Chartz", "future")]:
            age_ids = get_folder_ids(service, name=age_name, parent=ids[0])
            if not has_unique_folder(age_ids, age_name, folder_name): return None
            separated_ids[f'{abbr}_{age}'] = age_ids[0]

    return separated_ids

# Search folder for specific files/folders
def get_drive_files(service, id, file_types=None, files_only=True, folders_only=False, name=None, is_shortcut=False):
    output = []
    q = f'"{id}" in parents'
    if files_only: q += ' and not mimeType = "application/vnd.google-apps.folder"'
    if folders_only: q += ' and mimeType = "application/vnd.google-apps.folder"'
    if name: q += f' and name contains "{name}"'

    fields = f'files(id, name{", shortcutDetails" if is_shortcut else ""})'

    # Get all files/folders in the folder
    file_results = service.files().list(corpora="user",
                                        pageSize=1000,
                                        fields=fields,
                                        q=q,
                                        spaces="drive").execute()
    items = file_results.get('files', [])

    # Check that file extensions match
    for item in items:
        if not file_types or item.get('name')[-4:] in file_types:
            output.append(item)

    return output

# Return list of files in a given directory with a specific extension
def get_dir_files(dir, file_types):
    try:
        files = [ file for file in os.listdir(dir) if file[-4:] in file_types ]
        if len(files) == 0: raise OSError
        return files
    except OSError:
        print(f'ERROR: No supported files found in directory "{dir}"')
        sys.exit()

# A helper function that outputs some useful error messages
def has_unique_folder(res, target, parent):
    if res == None or len(res) == 0:
        print(f'ERROR: "{target}" folder not found in "{parent}" directory, please check Google Drive')
        return False
    elif len(res) > 1:
        print(f'ERROR: Multiple instances of "{target}" folder found in "{parent}" directory, please check Google Drive')
        print(f'Double-check the Trash section of Google Drive, as it still belongs to the "{parent}" folder')
        return False
    else:
        return True


from num2words import num2words

from odoo import models, tools

CURRENCY_NAME = {"Dollars": "ดอลลาร์", "Euros": "ยูโร", "Cents": "เซนต์"}

class Currency(models.Model):
    _inherit = "res.currency"

    def _convert_currency_name_hook(self, label):
        """Hooks to add currency translate.
        Currently, there are 2 currency are USD and EUR"""
        return CURRENCY_NAME[label]

    def amount_to_text(self, amount):
        self.ensure_one()
        lang_code = self.env.context.get("lang") or False
        if lang_code != "th_TH":
            return super().amount_to_text(amount)

        def _num2words(number, lang):
            try:
                return num2words(number, lang=lang).title()
            except NotImplementedError:
                return num2words(number, lang="en").title()

        formatted = "%.{}f".format(self.decimal_places) % amount
        parts = formatted.partition(".")
        integer_value = int(parts[0])
        fractional_value = int(parts[2] or 0)
        lang = (
            self.env["res.lang"]
            .with_context(active_test=False)
            .search([("code", "=", lang_code)])
        )
        # Thai Text with Thai Currency
        if self.name == "THB":
            try:
                return num2words(amount, to="currency", lang=lang.iso_code)
            except NotImplementedError:
                return num2words(amount, to="currency", lang="en")
        # Thai Text with Foreign currency
        currency_unit_label = self._convert_currency_name_hook(self.currency_unit_label)
        amount_words = tools.ustr("{amt_value}{amt_word}").format(
            amt_value=_num2words(integer_value, lang=lang.iso_code),
            amt_word=currency_unit_label,
        )
        if not self.is_zero(amount - integer_value):
            currency_subunit_label = self._convert_currency_name_hook(
                self.currency_subunit_label
            )
            amount_words += tools.ustr("{amt_value}{amt_word}").format(
                amt_value=_num2words(fractional_value, lang=lang.iso_code),
                amt_word=currency_subunit_label,
            )
        return amount_words


import pathlib
import numpy as np
from scipy.interpolate import RegularGridInterpolator

class Ppm():

    def __init__(self):
        self.data = None
        self.ijks = None
        self.normals = None
        self.ijk_interpolator = None
        self.normal_interpolator = None
        self.data_header = None
        self.valid = False
        self.error = "no error message set"

    def createErrorPpm(err):
        ppm = Ppm()
        ppm.error = err
        return ppm

    no_data = (0.,0.,0.)

    # lijk (layer ijk) is in layer's global coordinates
    def layerIjksToScrollIjks(self, lijks):
        print("litsi")
        if self.data is None:
            print("litsi no data")
            return lijks
        '''
        li,lj,lk = lijk
        if li < 0 or lj < 0:
            return Ppm.no_data
        if li >= self.width:
            return Ppm.no_data
        if lj >= self.height:
            return Ppm.no_data
        '''
        # sijk = np.zeros((lijk.shape), dtype=lijk.dtype)
        # sijks = self.ijk_interpolator(lijks[:,0:2])
        ijs = lijks[:,(2,0)]
        ks = lijks[:,1,np.newaxis]
        sijks = self.ijk_interpolator(ijs)
        norms = self.normal_interpolator(ijs)
        print(lijks.shape, sijks.shape, norms.shape, ks.shape)
        sijks += norms*(ks-32)
        return sijks

    def loadData(self):
        if self.data is not None:
            rturn
        print("reading data from %s for %s"%(str(self.path), self.name))

        fstr = str(self.path)
        print("reading ppm data for", self.path)
        if not self.path.exists():
            err="ppm file %s does not exist"%fstr
            print(err)
            return Ppm.createErrorPpm(err)

        try:
            fd = self.path.open("rb")
        except Exception as e:
            err="Failed to open ppm file %s: %s"%(fstr, e)
            print(err)
            return Ppm.createErrorPpm(err)

        try:
            bdata = fd.read()
        except Exception as e:
            err="Failed to read ppm file %s: %s"%(fstr, e)
            print(err)
            return Ppm.createErrorPpm(err)

        index = bdata.find(b'<>\n')
        if index < 0:
            err="Ppm file %s does not have a header"%fstr
            print(err)
            return Ppm.createErrorPpm(err)

        bdata = bdata[index+3:]
        lbd = len(bdata)
        height = self.height
        width = self.width
        le = height*width*8*6
        if lbd != le:
            err="Ppm file %s expected %d bytes of data, got %d"%(fstr, le, lbd)
            print(err)
            return Ppm.createErrorPpm(err)

        raw = np.frombuffer(bdata, dtype=np.float64)
        self.data = np.reshape(raw, (height,width,6))
        self.ijks = self.data[:,:,:3]
        self.normals = self.data[:,:,3:]
        print(self.ijks.shape, self.normals.shape)
        # print(self.ijks[0,0,:],self.normals[0,0,:])
        # print(self.ijks[3000,3000,:],self.normals[3000,3000,:])
        ii = np.arange(height)
        jj = np.arange(width)
        self.ijk_interpolator = RegularGridInterpolator((ii, jj), self.ijks, fill_value=0., bounds_error=False)
        self.normal_interpolator = RegularGridInterpolator((ii, jj), self.normals, fill_value=0., bounds_error=False)

    # reads and loads the header of the ppm file
    def loadPpm(filename):
        fstr = str(filename)
        print("reading ppm header for", filename)
        if not filename.exists():
            err="ppm file %s does not exist"%fstr
            print(err)
            return Ppm.createErrorPpm(err)

        try:
            fd = filename.open("rb")
        except Exception as e:
            err="Failed to open ppm file %s: %s"%(fstr, e)
            print(err)
            return Ppm.createErrorPpm(err)

        try:
            bstr = fd.read(200)
        except Exception as e:
            err="Failed to read ppm file %s: %s"%(fstr, e)
            print(err)
            return Ppm.createErrorPpm(err)

        index = bstr.find(b'<>\n')
        if index < 0:
            err="Ppm file %s does not have a header"%fstr
            print(err)
            return Ppm.createErrorPpm(err)

        hstr = bstr[:index+3].decode('utf-8')
        lines = hstr.split('\n')
        hdict = {}
        for line in lines:
            words = line.split()
            if len(words) != 2:
                continue
            name = words[0]
            value = words[1]
            if name[-1] != ':':
                continue
            name = name[:-1]
            hdict[name] = value
        for name in ["width", "height"]:
            if name not in hdict:
                err="Ppm file %s missing \"%s\" in header"%(fstr, name)
                print(err)
                return Ppm.createErrorPpm(err)

        try:
            width = int(hdict["width"])
        except Exception as e:
            err="Ppm file %s could not parse width value \"%s\" in header"%(fstr, hdict["width"])
            print(err)
            return Ppm.createErrorPpm(err)

        try:
            height = int(hdict["height"])
        except Exception as e:
            err="Ppm file %s could not parse height value \"%s\" in header"%(fstr, hdict["height"])
            print(err)
            return Ppm.createErrorPpm(err)

        expected = {
                "dim": "6",
                "ordered": "true",
                "type": "double",
                "version": "1",
                }

        for name, value in expected.items():
            if name not in hdict:
                err = "Ppm file %s missing \"%s\" from header"%(fstr, name)
                print(err)
                return Ppm.createErrorPpm(err)
            if hdict[name] != expected[name]:
                err = "Ppm file %s expected value of \"%s\" for \"%s\" in header; got %s"%(fstr, expected[name], name, hdict[name])
                print(err)
                return Ppm.createErrorPpm(err)

        ppm = Ppm()
        ppm.valid = True
        ppm.height = height
        ppm.width = width
        ppm.path = filename
        ppm.name = filename.stem
        print("created ppm %s width %d height %d"%(ppm.name, ppm.width, ppm.height))
        return ppm


from rest_framework import serializers
from users.models import User
from django.contrib.auth import get_user_model
from game.models import LeaderBoard, Round
from game.serializers import LeaderBoardSerializer

class CreateUserSerializer(serializers.ModelSerializer):

    class Meta:
        model = User
        extra_kwargs = {
            'password': {'write_only': True},
            'date_joined': {'read_only': True},
        }
        fields = (
            'id',
            'email',
            'username',
            'date_joined',
            'password',
            'is_superuser',
        )

    def create(self, validated_data):
        user = get_user_model()

        if validated_data.pop('is_superuser', False):
            return user.objects.create_superuser(**validated_data)
        else:
            user_instance = user.objects.create_user(**validated_data)
            first_round = Round.objects.filter(number=1).first()
            LeaderBoard.objects.create(
                user = user_instance,
                round = first_round
            )
            return user_instance

class LeaderBoardListing(serializers.RelatedField):

    obj = {}

    def to_representation(self, value):
        self.obj['round_id'] = value.id
        return self.obj

class UserSerializer(serializers.ModelSerializer):

    current_round = serializers.ReadOnlyField(source='leaderboard.round.number', read_only=True)
    date_joined = serializers.DateTimeField(format="%Y-%m-%d")

    class Meta:
        model = User
        extra_kwargs = {
            'email': {'read_only': True},
            'id': {'read_only': True},
            'date_joined': {'read_only': True},
        }
        fields = (
            'id',
            'first_name',
            'username',
            'avatar',
            'email',
            'auth_token',
            'date_joined',
            'current_round',
        )

class AccessTokenSerializer(serializers.Serializer):

    access_token = serializers.CharField(label="Access Token")


import requests
import bs4
import re
import json
import urllib.parse
import threading

baseUrl = "https://www.luogu.com.cn/problem/list?ProblemListParams&page="

def requestToLuogu(url):
    headers = {
        "authority":"www.luogu.com.cn",
        "method":"GET",
        "path":"/problem/P8000",
        "scheme":"https",
        "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
        "Accept-Encoding":"gzip, deflate, br",
        "Accept-Language":"zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
        "Cookie":"__client_id=a9a7f6dc9a1733e681d82f395f7d12d740413ff7; _uid=1088102; C3VK=195332",
        "Referer":"https://www.luogu.com.cn/",
        #"Sec-Ch-Ua":"Chromium";v="116", "Not)A;Brand";v="24", "Microsoft Edge";v="116"
        #Sec-Ch-Ua-Mobile:?0
        #Sec-Ch-Ua-Platform:"Windows"
        #Sec-Fetch-Dest:document
        #Sec-Fetch-Mode:navigate
        #Sec-Fetch-Site:same-origin
        #Sec-Fetch-User:?1
        "Upgrade-Insecure-Requests":"1",
        "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Edg/116.0.1938.76"
    }
    r = requests.get(url = url,headers = headers)
    #用Unicode格式解码
    #r.encoding = 'unicode_escape'
    return r

def analysis(text):
    text = str(urllib.parse.unquote(text,encoding ='unicode_escape'))
    #bs = bs4.BeautifulSoup(text,"html.parser")
    #core = bs.find("script")
    #print(text)
    core = re.findall('decodeURIComponent\("(.*?)"\)\)', text)[0]
    print(str(core))
    return core

def save(soup,i):
    content = str(soup)
    cfilename = "Json\\content"+str(i)+".json"
    content = content.replace('\\', '\\\\')
    #json.loads(content)
    file = open(cfilename,"w",encoding="utf-8")

    for d in content:
        file.writelines(d)
    file.close()

def CGJmain(JIndex):
        r = requestToLuogu(baseUrl+str(JIndex))
        soup = analysis(r.text)
        save(soup,JIndex)
def TCGJmain():
    threads = []
    t1 = threading.Thread(target = CGJmain,args=(0,))
    threads.append(t1)
    t2 = threading.Thread(target = CGJmain,args=(1,))
    threads.append(t2)
    t3 = threading.Thread(target = CGJmain,args=(2,))
    threads.append(t3)
    t4 = threading.Thread(target = CGJmain,args=(3,))
    threads.append(t4)
    t5 = threading.Thread(target = CGJmain,args=(4,))
    threads.append(t5)
    t6 = threading.Thread(target = CGJmain,args=(5,))
    threads.append(t6)
    t7 = threading.Thread(target = CGJmain,args=(6,))
    threads.append(t7)
    t8 = threading.Thread(target = CGJmain,args=(7,))
    threads.append(t8)
    t9 = threading.Thread(target = CGJmain,args=(8,))
    threads.append(t9)
    t10 = threading.Thread(target = CGJmain,args=(9,))
    threads.append(t10)
    for t in threads:
         t.start()


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import cvxpy as cp
import numpy as np
import matplotlib.pyplot as plt
import scipy.optimize
from scipy.optimize import curve_fit
from cnn import *

class SliceTuner:
    def __init__(self, train, val, val_data_dict, data_num_array, num_class, add_data_dict):
        """
        Args:
            train: Training data and label
            val: Valiation data and label
            val_data_dict: Validation data per each slice
            data_num_array: Initial slice sizes
            num_class: Number of class
            add_data_dict: Data assumed to be collected
        """

        self.train = train
        self.val = val
        self.val_data_dict = val_data_dict
        self.data_num_array = data_num_array
        self.add_data_dict = add_data_dict

        self.num_class = num_class
        self.loss_output = []
        self.slice_num = []

    def selective_collect(self, budget, k, cost_func, Lambda, num_iter, slice_desc, strategy="one-shot", show_figure=False):
        """
        Selecitve data collection function to determin how much data collection is needed for each slice
        given a budget in order to optimzie both accuracy and fairness.

        Args:
            budget: Data collection budget
            k: Number of subsets of data to fit a learning curve
            cost_func: Represents the effort to collect an example for a slice
            Lambda: Balancing term between loss and unfairness
            num_iter: Number of training times
            strategy: Strategy for updating the limit T (e.g., one-shot, aggressive, linear, conservative)
            show_figure: Plot the learning curve if show_figure is True
            slice_desc: Slice description (e.g., Slice: Shirt)
        """

        self.budget = budget
        self.Lambda = Lambda
        self.cost_func = cost_func

        initial_k = 200
        num_k_ = initial_k + np.arange(0, k) * (len(self.train[0]) - initial_k)/ (k-1)
        num_k = [int(i) for i in num_k_]
        print(num_k)

        self.T = 1
        self.train_on_subsets(num_k, num_iter)
        IR = self.get_imbalance_ratio(self.data_num_array)

        while self.budget > 0:
            num_examples = self.one_shot(slice_desc, show_figure)
            after_IR = self.get_imbalance_ratio(self.data_num_array + num_examples)
            if strategy != "one-shot" and abs(after_IR - IR) > self.T:
                target_ratio = IR + self.T * np.sign(after_IR - IR)
                change_ratio = self.get_change_ratio(self.data_num_array, num_examples, target_ratio)
                num_examples = [int(num_examples[i] * change_ratio) for i in range(self.num_class)]

            self.train_after_collect_data(num_examples, num_iter)
            self.data_num_array = [self.data_num_array[i] + num_examples[i] for i in range(self.num_class)]
            self.budget = self.budget - np.sum(np.add(np.multiply(num_examples, self.cost_func), 0.5).astype(int))
            self.increase_limit(strategy)
            IR = after_IR

            print("======= Collect Data =======")
            print(num_examples)
            print("Total Cost: %s, Remaining Budget: %s"
                  % (np.sum(np.add(np.multiply(num_examples, self.cost_func), 0.5).astype(int)), self.budget))

        print("======= Performance =======")
        print("Strategy: %s, C: %s, Budget: %s" % (strategy, self.Lambda, budget))
        print("Loss: %.5f, Average EER: %.5f, Max EER: %.5f\n" % tuple(self.show_performance()))

    def train_after_collect_data(self, num_examples, num_iter):
        """ Trains the model after we collect num_examples of data

        Args:
            num_examples: Number of examples to collect per slice
            num_iter: Number of training times
        """

        self.collect_data(num_examples)
        for i in range(num_iter):
            network = CNN(self.train[0], self.train[1], self.val[0], self.val[1], self.val_data_dict,
                          self.batch_size, epoch=500, lr = 0.001, num_class = self.num_class)
            loss_dict, slice_num = network.cnn_train()

            for j in range(self.num_class):
                if i == 0:
                    self.loss_output[j].append(loss_dict[j] / num_iter)
                    self.slice_num[j].append(slice_num[j])
                else:
                    self.loss_output[j][-1] += (loss_dict[j] / num_iter)

    def collect_data(self, num_examples):
        """
        Collects num_examples of data from add_data_dict
        add_data_dict could be changed to any other data collection tool

        Args:
            num_examples: Number of examples to collect per slice
        """

        def shuffle(data, label):
            shuffle = np.arange(len(data))
            np.random.shuffle(shuffle)
            data = data[shuffle]
            label = label[shuffle]
            return data, label

        train_data = self.train[0]
        train_label = self.train[1]
        for i in range(self.num_class):
            train_data = np.concatenate((train_data, self.add_data_dict[i][0][:num_examples[i]]), axis=0)
            train_label = np.concatenate((train_label, self.add_data_dict[i][1][:num_examples[i]]), axis=0)
            self.add_data_dict[i]= (np.concatenate((self.add_data_dict[i][0][num_examples[i]:], self.add_data_dict[i][0][:num_examples[i]]), axis=0),
                              np.concatenate((self.add_data_dict[i][1][num_examples[i]:], self.add_data_dict[i][1][:num_examples[i]]), axis=0))

        self.train = (train_data, train_label)

    def train_on_subsets(self, num_k, num_iter):
        """
        Given the slices, we generate for each slice k random subsets of data to fit a power-law curve.

        Args:
            num_k: Subsets of the train set with different sizes
            num_iter: Number of training times
        """

        self.batch_size = self.train[0].shape[0]

        for i in range(self.num_class):
            self.loss_output.append([0] * len(num_k))
            self.slice_num.append([])

        for k in range(len(num_k)):
            for i in range(num_iter):
                network = CNN(self.train[0][:num_k[k]], self.train[1][:num_k[k]], self.val[0], self.val[1], self.val_data_dict,
                              self.batch_size, epoch=500, lr = 0.001, num_class = self.num_class)
                loss_dict, slice_num = network.cnn_train()

                for j in range(self.num_class):
                    self.loss_output[j][k] += (loss_dict[j] / num_iter)
                    if i == 0:
                        self.slice_num[j].append(slice_num[j])

    def one_shot(self, slice_desc, show_figure):
        """
        Fits the learning curve, and solves the optimization problem using entire budget

        Args:
            slice_desc: Array for slice description (e.g., Slice: White_Male)
            show_figure: If True, show learning curves for all slices

        Return: Number of examples to collect for the slices within the budget
        """

        A, B, estimate_loss = self.fit_learning_curve(slice_desc, show_figure)
        return self.op_func(A, B, estimate_loss)

    def fit_learning_curve(self, slice_desc, show_figure):
        """
        Fits the learning curve, assuming power-law form

        Args:
            slice_desc: Array for slice description (e.g., Slice: White_Male)
            show_figure: If True, show learning curves for all slices

        Returns:
            A: Exponent of power-law equation
            B: of power-law equation
            estimate_loss: estimated loss on given data using power-law equation
        """

        def weight_list(weight):
            w_list = []
            for i in weight:
                w_list.append(1/(i**0.5))
            return w_list

        def power_law(x, a, b):
            return (b*x**(-a))

        A = []
        B = []
        estimate_loss = []
        for i in range(self.num_class):
            xdata = np.linspace(self.slice_num[i][0], self.slice_num[i][-1], 1000)
            sigma = weight_list(self.slice_num[i])
            popt, pcov = curve_fit(power_law, self.slice_num[i], self.loss_output[i], sigma=sigma, absolute_sigma=True)

            A.append(-popt[0])
            B.append(popt[1])
            estimate_loss.append(popt[1] * (self.data_num_array[i] ** (-popt[0])))

            if show_figure == True:
                plt.figure(1, figsize=(12,8))
                plt.plot(self.slice_num[i], self.loss_output[i], 'o-', linewidth=1.0, markersize=4, label=slice_desc[i])
                plt.plot(xdata, power_law(xdata, *popt), linewidth=2.0, label='$y={%0.3f}x^{-{%0.3f}}$' % (popt[1], popt[0]))

                plt.tick_params(labelsize=20)
                plt.xlabel('Number of training examples', fontsize=25)
                plt.ylabel('Validation Loss', fontsize=25)
                plt.legend(prop={'size':25})

                plt.tight_layout()
                plt.show()

        return A, B, estimate_loss

    def increase_limit(self, strategy):
        """
        Updates the limit T according to the strategy
            Aggressive: For each iteration, multiply T by 2
            Linear: For each iteration, increase T by 1
            Conservative: Leave T as a constant
        """

        if strategy == "aggressive":
            self.T = self.T * 2
        elif strategy == "linear":
            self.T = self.T + 1

    def get_imbalance_ratio(self, data_array):
        """
        Calculate imbalance ratio of the data

        Args:
            data_array: Number of data for slices

        Return: Imbalance ratio
        """

        return max(data_array) / min(data_array)

    def get_change_ratio(self, data_array, num_examples, target_ratio):
        """
        If the imbalance ratio change would exceed T when using the entire budget,
        limit the number of examples collected by the target_ratio by solving a non-linear equation

        Args:
            data_array: Number of data for slices
            num_examples: Number of examples to collect per slice
            target_ratio: Imbalance ratio limit

        Return: ratio that makes the imbalance ratio change would not exceed target ratio
        """

        def F(x, num, add, target):
            func1 = max([int(num_examples[i]*x) + num[i] for i in range(self.num_class)])
            func2 = min([int(num_examples[i]*x) + num[i] for i in range(self.num_class)])
            return func1 - target * func2

        ratio = scipy.optimize.fsolve(F, x0=(0.5), args=(data_array, num_examples, target_ratio))
        if ratio < 0:
            ratio =  scipy.optimize.fsolve(F, x0=(1.0), args=(data_array, num_examples, target_ratio))
        elif ratio > 1:
            ratio = scipy.optimize.fsolve(F, x0=(0.01), args=(data_array, num_examples, target_ratio))

        return ratio

    def op_func(self, A, B, estimate_loss):
        """
        Solves the convex optimzation problem

        Args:
            A: Exponent of power-law equation
            B: of power-law equation
            estimate_loss: estimated loss on given data using power-law equation

        Return: Number of examples to collect for the slices within the budget
        """

        x = cp.Variable(self.num_class, integer=True)
        for i in range(self.num_class):
            loss = cp.multiply(B[i], cp.power((x[i]+self.data_num_array[i]), A[i]))
            counter_loss = (np.sum(estimate_loss) - estimate_loss[i]) / (self.num_class - 1)
            if i==0:
                ob_func = loss + self.Lambda * cp.maximum(0, (loss / counter_loss) - 1)
            else:
                ob_func += loss + self.Lambda * cp.maximum(0, (loss / counter_loss) - 1)

        constraints = [cp.sum(cp.multiply(x, self.cost_func)) <= self.budget] + [x>=0]
        objective = cp.Minimize(ob_func)
        prob = cp.Problem(objective, constraints)
        prob.solve(solver="ECOS_BB")
        return np.add(x.value, 0.5).astype(int)

    def show_performance(self):
        """ Average validation loss, Average equalized error rate(Avg. EER), Maximum equalized error rate (Max. EER) """

        final_loss = []
        num = 0
        max_eer = 0
        avg_eer =0

        for i in range(self.num_class):
            final_loss.append(self.loss_output[i][-1])

        for i in range(self.num_class):
            diff_eer = ((np.sum(final_loss) - final_loss[i]) / (self.num_class-1) - final_loss[i]) * (-1)
            if diff_eer > 0:
                if max_eer < diff_eer:
                    max_eer = diff_eer

                avg_eer += diff_eer
                num += 1

        avg_eer = avg_eer / num
        return np.average(final_loss), avg_eer, max_eer


from __future__ import annotations

from typing import Any
from unittest.mock import Mock

from coola.formatters.base import BaseFormatter
from coola.summarizers.base import BaseSummarizer
from coola.utils import check_torch, is_torch_available

if is_torch_available():
    import torch
else:
    torch = Mock()  # pragma: no cover

class TensorFormatter(BaseFormatter[torch.Tensor]):
    r"""Implement a formatter for ``torch.Tensor``.

    Args:
    ----
        show_data (bool, optional): If ``True``, the returned string
            is the default string representation (``repr``).
            If ``False``, the returned string only contains the tensor
            metadata. Default: ``False``
    """

    def __init__(self, show_data: bool = False) -> None:
        check_torch()
        self._show_data = bool(show_data)

    def __repr__(self) -> str:
        return f"{self.__class__.__qualname__}(show_data={self._show_data})"

    def clone(self) -> TensorFormatter:
        return self.__class__(show_data=self._show_data)

    def equal(self, other: Any) -> bool:
        if not isinstance(other, self.__class__):
            return False
        return self._show_data == other._show_data

    def format(
        self, summarizer: BaseSummarizer, value: torch.Tensor, depth: int = 0, max_depth: int = 1
    ) -> str:
        if self._show_data:
            return repr(value)
        return " | ".join(
            [
                f"{type(value)}",
                f"shape={value.shape}",
                f"dtype={value.dtype}",
                f"device={value.device}",
            ]
        )

    def load_state_dict(self, state: dict) -> None:
        self._show_data = state["show_data"]

    def state_dict(self) -> dict:
        return {"show_data": self._show_data}

    def get_show_data(self) -> bool:
        r"""Indicates if the tensor data or metadata are shown.

        Returns:
        -------
            bool: ``True`` if the tensor data are shown,
                ``False`` if the tensor metadata are shown.

        Example usage:

        .. code-block:: pycon

            >>> from coola.formatters import TensorFormatter
            >>> formatter = TensorFormatter()
            >>> formatter.get_show_data()
            False
        """
        return self._show_data

    def set_show_data(self, show_data: bool) -> None:
        r"""Set if the tensor data or metadata are shown.

        Args:
        ----
            show_data (bool): ``True`` if the tensor data are shown,
                ``False`` if the tensor metadata are shown.

        Raises:
        ------
            TypeError if ``show_data`` is not an boolean.

        Example usage:

        .. code-block:: pycon

            >>> from coola.formatters import TensorFormatter
            >>> formatter = TensorFormatter()
            >>> formatter.set_show_data(True)
            >>> formatter.get_show_data()
            True
        """
        if not isinstance(show_data, bool):
            raise TypeError(
                "Incorrect type for show_data. Expected bool value but received {show_data}"
            )
        self._show_data = show_data


import numpy as np
import matplotlib.pyplot as plt
import matplotlib
import matplotlib.gridspec as gridspec
from scipy import signal

matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['svg.fonttype'] = 'none'

#%% script params

num_axons = 3
num_dendrites = 9

axon_colors = ['magenta', 'teal', 'purple']

num_spikes_per_axon = 3
experiment_time_ms  = 300

tau_rise_range  = [1,9]
tau_decay_range = [5,30]

v_reset     = -80
v_threshold = -55
current_to_voltage_mult_factor = 4
refreactory_time_constant = 25

save_figures = False
all_file_endings_to_use = ['.png', '.pdf', '.svg']

figure_folder = '/filter_and_fire_neuron/saved_figures/'

#%% helper functions

def create_single_PSP_profile(tau_rise, tau_decay, temporal_filter_length=50):

    if tau_rise >= tau_decay:
        tau_decay = tau_rise + 5

    exp_r = signal.exponential(M=temporal_filter_length, center=0, tau=tau_rise , sym=False)
    exp_d = signal.exponential(M=temporal_filter_length, center=0, tau=tau_decay, sym=False)

    post_syn_potential = exp_d - exp_r
    post_syn_potential /= post_syn_potential.max()

    return post_syn_potential

def construct_normlized_synaptic_filter(tau_rise_vec, tau_decay_vec):

    num_synapses = tau_rise_vec.shape[0]
    temporal_filter_length = int(7 * tau_decay_vec.max()) + 1

    syn_filter = np.zeros((num_synapses, temporal_filter_length))

    for k, (tau_r, tau_d) in enumerate(zip(tau_rise_vec, tau_decay_vec)):
        syn_filter[k,:] = create_single_PSP_profile(tau_r, tau_d, temporal_filter_length=temporal_filter_length)

    return syn_filter

def simulate_filter_and_fire_cell_training(presynaptic_input_spikes, synaptic_weights, tau_rise_vec, tau_decay_vec,
                                           refreactory_time_constant=20, v_reset=-75, v_threshold=-55, current_to_voltage_mult_factor=2):

    temporal_filter_length = int(5 * refreactory_time_constant) + 1
    refreactory_filter = signal.exponential(M=temporal_filter_length,center=0,tau=refreactory_time_constant,sym=False)[np.newaxis,:]

    # padd input and get all synaptic filters
    normlized_syn_filter = np.flipud(construct_normlized_synaptic_filter(tau_rise_vec, tau_decay_vec))
    padded_input = np.hstack((np.zeros(normlized_syn_filter.shape), presynaptic_input_spikes))

    # calc local currents
    local_normlized_currents = np.zeros(presynaptic_input_spikes.shape, dtype=np.float16)
    for k in range(normlized_syn_filter.shape[0]):
        local_normlized_currents[k] = signal.convolve(padded_input[k], normlized_syn_filter[k], mode='valid')[1:]

    # multiply by weights to get the somatic current
    soma_current = signal.convolve(local_normlized_currents, synaptic_weights, mode='valid')

    # simulate the cell
    soma_voltage = v_reset + current_to_voltage_mult_factor * soma_current.ravel()
    output_spike_times_in_ms = []
    # after a spike inject current that is exactly required to bring the cell back to v_reset (this current slowly decays)
    for t in range(len(soma_voltage)):
        # after a spike inject current that is exactly required to bring the cell back to v_reset (this current slowly decays)
        if (soma_voltage[t] > v_threshold) and ((t + 1) < len(soma_voltage)):
            t_start = t + 1
            t_end = min(len(soma_voltage), t_start + temporal_filter_length)
            soma_voltage[t_start:t_end] -= (soma_voltage[t + 1] - v_reset) * refreactory_filter.ravel()[:(t_end - t_start)]
            output_spike_times_in_ms.append(t)

    return local_normlized_currents, soma_voltage, output_spike_times_in_ms

def add_offset_for_plotting(traces_matrix, offset_size=1.1):

    traces_matrix_with_offset = offset_size * np.kron(np.arange(traces_matrix.shape[0])[:,np.newaxis], np.ones((1,traces_matrix.shape[1])))
    traces_matrix_with_offset = traces_matrix_with_offset + traces_matrix

    return traces_matrix_with_offset

#%% simulate the cell

connections_per_axon = int(num_dendrites / num_axons)
num_synapses = num_dendrites

tau_rise_vec  = np.linspace(tau_rise_range[0] , tau_rise_range[1] , num_synapses)[:,np.newaxis]
tau_decay_vec = np.linspace(tau_decay_range[0], tau_decay_range[1], num_synapses)[:,np.newaxis]

# synapse learnable parameters
synaptic_weights_vec = 1.0 + 0.1 * np.random.uniform(size=(num_synapses, 1))

synaptic_filters = construct_normlized_synaptic_filter(tau_rise_vec, tau_decay_vec)

#%% run once

# generate sample input
stimulus_duration_ms = experiment_time_ms

axon_input_spike_train = np.zeros((num_axons, stimulus_duration_ms))
for k in range(num_axons):
    curr_axon_spike_times = 20 + np.random.randint(stimulus_duration_ms -80, size=num_spikes_per_axon)
    axon_input_spike_train[k,curr_axon_spike_times] = 1.0

presynaptic_input_spikes = np.kron(np.ones((connections_per_axon,1), dtype=bool), axon_input_spike_train).astype(bool)

# simulate F&F cell with normlized currents
local_normlized_currents, soma_voltage, output_spike_times_in_ms = simulate_filter_and_fire_cell_training(presynaptic_input_spikes,
                                                                                synaptic_weights_vec, tau_rise_vec, tau_decay_vec,
                                                                                refreactory_time_constant=refreactory_time_constant,
                                                                                v_reset=v_reset, v_threshold=v_threshold,
                                                                                current_to_voltage_mult_factor=current_to_voltage_mult_factor)

local_normlized_currents = np.flipud(local_normlized_currents)

soma_voltage_with_spikes = soma_voltage
soma_voltage_with_spikes[output_spike_times_in_ms] = -25

max_local_added_voltage = add_offset_for_plotting(local_normlized_currents).T.max()

print('running once')

#%% run again until we have at least 1 spike

while len(output_spike_times_in_ms) != 1 or max_local_added_voltage > 10.15:
    axon_input_spike_train = np.zeros((num_axons, stimulus_duration_ms))
    for k in range(num_axons):
        curr_axon_spike_times = 30 + np.random.randint(stimulus_duration_ms -60, size=num_spikes_per_axon)
        axon_input_spike_train[k,curr_axon_spike_times] = 1.0

    presynaptic_input_spikes = np.kron(np.ones((connections_per_axon,1), dtype=bool), axon_input_spike_train).astype(bool)

    # simulate F&F cell with normlized currents
    local_normlized_currents, soma_voltage, output_spike_times_in_ms = simulate_filter_and_fire_cell_training(presynaptic_input_spikes,
                                                                                synaptic_weights_vec, tau_rise_vec, tau_decay_vec,
                                                                                refreactory_time_constant=refreactory_time_constant,
                                                                                v_reset=v_reset, v_threshold=v_threshold,
                                                                                current_to_voltage_mult_factor=current_to_voltage_mult_factor)

    local_normlized_currents = np.flipud(local_normlized_currents)

    soma_voltage_with_spikes = soma_voltage
    soma_voltage_with_spikes[output_spike_times_in_ms] = -15

    max_local_added_voltage = add_offset_for_plotting(local_normlized_currents).T.max()

print('there is at least 1 spike')

#%% run untill we have a changed enough spike location

min_spike_time_diff = 35
max_local_added_voltage = 0

while True:

    output_spike_times_in_ms =[]

    # generate input axons with a single spike
    while len(output_spike_times_in_ms) != 1 or max_local_added_voltage > 10.15:
        axon_input_spike_train = np.zeros((num_axons, stimulus_duration_ms))
        for k in range(num_axons):
            curr_axon_spike_times = 30 + np.random.randint(stimulus_duration_ms -60, size=num_spikes_per_axon)
            axon_input_spike_train[k,curr_axon_spike_times] = 1.0

        presynaptic_input_spikes = np.kron(np.ones((connections_per_axon,1), dtype=bool), axon_input_spike_train).astype(bool)

        # simulate F&F cell with normlized currents
        local_normlized_currents, soma_voltage, output_spike_times_in_ms = simulate_filter_and_fire_cell_training(presynaptic_input_spikes,
                                                                                synaptic_weights_vec, tau_rise_vec, tau_decay_vec,
                                                                                refreactory_time_constant=refreactory_time_constant,
                                                                                v_reset=v_reset, v_threshold=v_threshold,
                                                                                current_to_voltage_mult_factor=current_to_voltage_mult_factor)

        local_normlized_currents = np.flipud(local_normlized_currents)

        soma_voltage_with_spikes = soma_voltage
        soma_voltage_with_spikes[output_spike_times_in_ms] = -15

        max_local_added_voltage = add_offset_for_plotting(local_normlized_currents).T.max()

        if len(output_spike_times_in_ms) == 1 and (output_spike_times_in_ms[0] > 200 or output_spike_times_in_ms[0] < 120):
            output_spike_times_in_ms = []

    print('there is at least 1 spike, spike times: ', output_spike_times_in_ms)

    # generate a weights change that will move this spikes by a minimum amount
    mult_vector = np.random.permutation([0.25,0.5,0.5,0.75,1.25,1.5,1.75,1.75,2.0])[:,np.newaxis]
    synaptic_weights_vec_2 = mult_vector * synaptic_weights_vec

    # make sure only the a the middle axon weights are changed (for non cluttered visualization)
    synaptic_weights_vec_2[0::3] = synaptic_weights_vec[0::3]
    synaptic_weights_vec_2[2::3] = synaptic_weights_vec[2::3]
    mult_vector[0::3] = 1
    mult_vector[2::3] = 1

    # simulate F&F cell with normlized currents
    local_normlized_currents_2, soma_voltage_2, output_spike_times_in_ms_2 = simulate_filter_and_fire_cell_training(presynaptic_input_spikes,
                                                                                synaptic_weights_vec_2, tau_rise_vec, tau_decay_vec,
                                                                                refreactory_time_constant=refreactory_time_constant,
                                                                                v_reset=v_reset, v_threshold=v_threshold,
                                                                                current_to_voltage_mult_factor=current_to_voltage_mult_factor)

    local_normlized_currents_2 = np.flipud(local_normlized_currents_2)

    soma_voltage_with_spikes_2 = soma_voltage_2
    soma_voltage_with_spikes_2[output_spike_times_in_ms_2] = -15

    max_local_added_voltage_2 = add_offset_for_plotting(local_normlized_currents_2).T.max()

    local_normlized_currents_2 = mult_vector * local_normlized_currents_2

    if len(output_spike_times_in_ms_2) == 1 and ((output_spike_times_in_ms[0] - output_spike_times_in_ms_2[0]) <= -min_spike_time_diff):
        break

print('changed weights such that the spike changed location for more than %d ms' %(min_spike_time_diff))

#%% Display Input Axons, Synaptic Filters and Local Voltage Traces "Slopily"

plt.close('all')
fig = plt.figure(figsize=(20,8))
plt.subplots_adjust(left=0.03, right=0.97, top=0.95, bottom=0.05, wspace=0.25, hspace=0.35)

plt.subplot(1,3,1);
for k, axon_color in enumerate(axon_colors):
    plt.plot(add_offset_for_plotting(np.flipud(axon_input_spike_train)).T[:,k], color=axon_color); plt.title('Input Axons', fontsize=24);

plt.subplot(1,3,2);
for k, axon_color in enumerate(axon_colors):
    plt.plot(add_offset_for_plotting(synaptic_filters).T[:100,k::num_axons], color=axon_color); plt.title('Synaptic Filters', fontsize=24);

plt.subplot(1,3,3);
for k, axon_color in enumerate(axon_colors):
    plt.plot(add_offset_for_plotting(local_normlized_currents).T[:,k::num_axons], color=axon_color); plt.title('Synaptic Contact Voltage Contribution', fontsize=24);
    plt.plot(add_offset_for_plotting(local_normlized_currents_2).T[:,k::num_axons], color=axon_color, linestyle='dashed');

#%% plot one below each other

plt.close('all')
plt.figure(figsize=(10,25))
gs_figure = gridspec.GridSpec(nrows=11,ncols=1)
gs_figure.update(left=0.04, right=0.95, bottom=0.05, top=0.95, wspace=0.1, hspace=0.8)

ax_axons          = plt.subplot(gs_figure[:3,:])
ax_local_voltages = plt.subplot(gs_figure[3:9,:])
ax_soma_voltage   = plt.subplot(gs_figure[9:,:])

ax_axons.plot(add_offset_for_plotting(axon_input_spike_train).T); ax_axons.set_title('Input Axons', fontsize=24);
ax_local_voltages.plot(add_offset_for_plotting(local_normlized_currents).T); ax_local_voltages.set_title('Synaptic Contact Voltage Contribution', fontsize=24);
ax_local_voltages.plot(add_offset_for_plotting(local_normlized_currents_2).T, linestyle='dashed');
ax_soma_voltage.plot(soma_voltage_with_spikes); ax_soma_voltage.set_title('Somatic Voltage', fontsize=24);
ax_soma_voltage.plot(soma_voltage_with_spikes_2, linestyle='dashed');

#%% Display The Full Figure

plt.close('all')
fig = plt.figure(figsize=(25,18))
gs_figure = gridspec.GridSpec(nrows=11,ncols=30)
gs_figure.update(left=0.04, right=0.95, bottom=0.05, top=0.92, wspace=0.2, hspace=0.7)

ax_axons         = plt.subplot(gs_figure[2:6,:9])
ax_syn_filters   = plt.subplot(gs_figure[:8,13:17])
ax_local_voltges = plt.subplot(gs_figure[:8,21:])
ax_soma_voltage  = plt.subplot(gs_figure[8:,21:])

for k, axon_color in enumerate(axon_colors):
    ax_axons.plot(add_offset_for_plotting(np.flipud(axon_input_spike_train)).T[:,k], color=axon_color, lw=3);
ax_axons.set_yticks([])
ax_axons.set_xticks([])
ax_axons.set_title('Input Axons', fontsize=20);
ax_axons.spines['top'].set_visible(False)
ax_axons.spines['bottom'].set_visible(False)
ax_axons.spines['left'].set_visible(False)
ax_axons.spines['right'].set_visible(False)

for k, axon_color in enumerate(axon_colors):
    ax_syn_filters.plot(add_offset_for_plotting(synaptic_filters).T[:100,k::num_axons], color=axon_color, lw=3);
ax_syn_filters.set_yticks([])
ax_syn_filters.set_xticks([])
ax_syn_filters.set_title('Synaptic Filters', fontsize=20);
ax_syn_filters.spines['top'].set_visible(False)
ax_syn_filters.spines['bottom'].set_visible(False)
ax_syn_filters.spines['left'].set_visible(False)
ax_syn_filters.spines['right'].set_visible(False)

for k, axon_color in enumerate(axon_colors):
    ax_local_voltges.plot(add_offset_for_plotting(local_normlized_currents).T[:,k::num_axons], color=axon_color, lw=3);
    ax_local_voltges.plot(add_offset_for_plotting(local_normlized_currents_2).T[:,k::num_axons], color=axon_color, ls='dashed', lw=2.5);
ax_local_voltges.set_xticks([])
ax_local_voltges.set_yticks([])
ax_local_voltges.set_title('Synaptic Contact Voltage Contribution', fontsize=20);
ax_local_voltges.spines['top'].set_visible(False)
ax_local_voltges.spines['bottom'].set_visible(False)
ax_local_voltges.spines['left'].set_visible(False)
ax_local_voltges.spines['right'].set_visible(False)

ax_soma_voltage.plot(soma_voltage_with_spikes, color='0.1', lw=3, alpha=0.85);
ax_soma_voltage.plot(soma_voltage_with_spikes_2, color='dodgerblue', lw=3);
ax_soma_voltage.set_yticks([])
ax_soma_voltage.set_xticks([])
ax_soma_voltage.set_title('Somatic Voltage', fontsize=20);
ax_soma_voltage.spines['top'].set_visible(False)
ax_soma_voltage.spines['bottom'].set_visible(False)
ax_soma_voltage.spines['left'].set_visible(False)
ax_soma_voltage.spines['right'].set_visible(False)

# save figure
if save_figures:
    figure_name = 'F&F_A_Introduction_Figure_1_%d' %(np.random.randint(2000))
    for file_ending in all_file_endings_to_use:
        if file_ending == '.png':
            fig.savefig(figure_folder + figure_name + file_ending, bbox_inches='tight')
        else:
            fig.savefig(figure_folder + figure_name + file_ending, bbox_inches='tight')

#%%


# coding=utf-8
"""test distillnation(Edition 2), not use."""

from __future__ import absolute_import, division, print_function
import argparse
import csv
import logging
import os
import random
import sys

import numpy as np
import torch
from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler,
                              TensorDataset)
from tqdm import tqdm, trange
from torch.nn import CrossEntropyLoss, MSELoss
from scipy.stats import pearsonr, spearmanr
from sklearn.metrics import matthews_corrcoef, f1_score
from ..transformer.file_utils import WEIGHTS_NAME, CONFIG_NAME
from ..Bi_LSTM.modeling import BiLSTM
from ..Bi_LSTM.tokenization import BiLstmTokenizer

csv.field_size_limit(sys.maxsize)

log_format = '%(asctime)s %(message)s'
logging.basicConfig(stream=sys.stdout, level=logging.INFO,
                    format=log_format, datefmt='%m/%d %I:%M:%S %p')
fh = logging.FileHandler('debug_layer_loss.log')
fh.setFormatter(logging.Formatter(log_format))
logging.getLogger().addHandler(fh)
logger = logging.getLogger()

oncloud = True
try:
    import moxing as mox
except:
    oncloud = False

class InputExample(object):
    """A single training/test example for simple sequence classification."""

    def __init__(self, guid, text_a, text_b=None, label=None):
        """Constructs a InputExample.

        Args:
            guid: Unique id for the example.
            text_a: string. The untokenized text of the first sequence. For single
            sequence tasks, only this sequence must be specified.
            text_b: (Optional) string. The untokenized text of the second sequence.
            Only must be specified for sequence pair tasks.
            label: (Optional) string. The label of the example. This should be
            specified for train and dev examples, but not for test examples.
        """
        self.guid = guid
        self.text_a = text_a
        self.text_b = text_b
        self.label = label

class InputFeatures(object):
    """A single set of features of data."""

    def __init__(self, input_ids, input_mask, segment_ids, label_id, seq_length=None):
        self.input_ids = input_ids
        self.input_mask = input_mask
        self.segment_ids = segment_ids
        self.seq_length = seq_length
        self.label_id = label_id

class DataProcessor(object):
    """Base class for data converters for sequence classification data sets."""

    def get_train_examples(self, data_dir):
        """Gets a collection of `InputExample`s for the train set."""
        raise NotImplementedError()

    def get_dev_examples(self, data_dir):
        """Gets a collection of `InputExample`s for the dev set."""
        raise NotImplementedError()

    def get_labels(self):
        """Gets the list of labels for this data set."""
        raise NotImplementedError()

    @classmethod
    def _read_tsv(cls, input_file, quotechar=None):
        """Reads a tab separated value file."""
        with open(input_file, "r", encoding="utf-8") as f:
            reader = csv.reader(f, delimiter="\t", quotechar=quotechar)
            lines = []
            for line in reader:
                if sys.version_info[0] == 2:
                    line = list(unicode(cell, 'utf-8') for cell in line)
                lines.append(line)
            return lines

class Sst2Processor(DataProcessor):
    """Processor for the SST-2 data set (GLUE version)."""

    def get_train_examples(self, data_dir):
        """See base class."""
        return self._create_examples(
            self._read_tsv(os.path.join(data_dir, "train.tsv")), "train")

    def get_dev_examples(self, data_dir):
        """See base class."""
        return self._create_examples(
            self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev")

    def get_aug_examples(self, data_dir):
        return self._create_examples(
            self._read_tsv(os.path.join(data_dir, "train_aug.tsv")), "aug")

    def get_labels(self):
        """See base class."""
        return ["0", "1"]

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for (i, line) in enumerate(lines):
            if i == 0:
                continue
            guid = "%s-%s" % (set_type, i)
            text_a = line[0]
            label = line[1]
            examples.append(
                InputExample(guid=guid, text_a=text_a, text_b=None, label=label))
        return examples

def convert_examples_to_features4lstm(examples, label_list, max_seq_length,
                                 tokenizer, output_mode):
    """Loads a data file into a list of `InputBatch`s."""

    label_map = {label: i for i, label in enumerate(label_list)}

    features = []
    for (ex_index, example) in enumerate(examples):
        if ex_index % 10000 == 0:
            logger.info("Writing example %d of %d" % (ex_index, len(examples)))

        tokens = tokenizer.tokenize(example.text_a)

        input_ids = tokenizer.convert_tokens_to_ids(tokens)
        seq_length = len(input_ids)
        # input_mask = [1] * len(input_ids)

        padding = [0] * (max_seq_length - len(input_ids))
        input_ids += padding
        # input_mask += padding

        assert len(input_ids) == max_seq_length
        # assert len(input_mask) == max_seq_length

        if output_mode == "classification":
            label_id = label_map[example.label]
        elif output_mode == "regression":
            label_id = float(example.label)
        else:
            raise KeyError(output_mode)

        if ex_index < 1:
            logger.info("*** Example ***")
            logger.info("guid: %s" % (example.guid))
            logger.info("tokens: %s" % " ".join(
                [str(x) for x in tokens]))
            logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
            # logger.info("input_mask: %s" % " ".join([str(x) for x in input_mask]))
            logger.info("label: {}".format(example.label))
            logger.info("label_id: {}".format(label_id))

        features.append(
            InputFeatures(input_ids=input_ids,
                          input_mask=None,
                          segment_ids=None,
                          label_id=label_id,
                          seq_length=seq_length))
    return features

def _truncate_seq_pair(tokens_a, tokens_b, max_length):
    """Truncates a sequence pair in place to the maximum length."""
    while True:
        total_length = len(tokens_a) + len(tokens_b)
        if total_length <= max_length:
            break
        if len(tokens_a) > len(tokens_b):
            tokens_a.pop()
        else:
            tokens_b.pop()

def simple_accuracy(preds, labels):
    return (preds == labels).mean()

def acc_and_f1(preds, labels):
    acc = simple_accuracy(preds, labels)
    f1 = f1_score(y_true=labels, y_pred=preds)
    return {
        "acc": acc,
        "f1": f1,
        "acc_and_f1": (acc + f1) / 2,
    }

def pearson_and_spearman(preds, labels):
    pearson_corr = pearsonr(preds, labels)[0]
    spearman_corr = spearmanr(preds, labels)[0]
    return {
        "pearson": pearson_corr,
        "spearmanr": spearman_corr,
        "corr": (pearson_corr + spearman_corr) / 2,
    }

def compute_metrics(task_name, preds, labels):
    assert len(preds) == len(labels)
    if task_name == "cola":
        return {"mcc": matthews_corrcoef(labels, preds)}
    elif task_name == "sst-2":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "mrpc":
        return acc_and_f1(preds, labels)
    elif task_name == "sts-b":
        return pearson_and_spearman(preds, labels)
    elif task_name == "qqp":
        return acc_and_f1(preds, labels)
    elif task_name == "mnli":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "mnli-mm":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "qnli":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "rte":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "wnli":
        return {"acc": simple_accuracy(preds, labels)}
    else:
        raise KeyError(task_name)

def get_tensor_data(output_mode, features):
    if output_mode == "classification":
        all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.long)
    elif output_mode == "regression":
        all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.float)

    all_seq_lengths = torch.tensor([f.seq_length for f in features], dtype=torch.long)
    all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
    all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
    all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)
    tensor_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids,
                                all_label_ids, all_seq_lengths)
    return tensor_data, all_label_ids

def result_to_file(result, file_name):
    with open(file_name, "a") as writer:
        logger.info("***** Eval results *****")
        for key in sorted(result.keys()):
            logger.info("  %s = %s", key, str(result[key]))
            writer.write("%s = %s\n" % (key, str(result[key])))

def do_eval(model, task_name, eval_dataloader,
            device, output_mode, eval_labels, num_labels):
    eval_loss = 0
    nb_eval_steps = 0
    preds = []

    for batch_ in tqdm(eval_dataloader, desc="Evaluating"):
        batch_ = tuple(t.to(device) for t in batch_)
        with torch.no_grad():
            input_ids, _, _, label_ids, seq_lengths = batch_
            _, logits, _ = model(input_ids, seq_lengths)

        # create eval loss and other metric required by the task
        if output_mode == "classification":
            loss_fct = CrossEntropyLoss()
            tmp_eval_loss = loss_fct(logits.view(-1, num_labels), label_ids.view(-1))
        elif output_mode == "regression":
            loss_fct = MSELoss()
            tmp_eval_loss = loss_fct(logits.view(-1), label_ids.view(-1))

        eval_loss += tmp_eval_loss.mean().item()
        nb_eval_steps += 1
        if len(preds) == 0:
            preds.append(logits.detach().cpu().numpy())
        else:
            preds[0] = np.append(
                preds[0], logits.detach().cpu().numpy(), axis=0)

    eval_loss = eval_loss / nb_eval_steps

    preds = preds[0]
    if output_mode == "classification":
        preds = np.argmax(preds, axis=1)
    elif output_mode == "regression":
        preds = np.squeeze(preds)
    result = compute_metrics(task_name, preds, eval_labels.numpy())
    result['eval_loss'] = eval_loss

    return result

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--data_dir",
                        default=None,
                        type=str,
                        required=True,
                        help="The input data dir. Should contain the .tsv files (or other data files) for the task.")
    parser.add_argument("--teacher_model",
                        default=None,
                        type=str,
                        help="The teacher model dir.")
    parser.add_argument("--student_model",
                        default=None,
                        type=str,
                        required=True,
                        help="The student model dir.")
    parser.add_argument("--task_name",
                        default=None,
                        type=str,
                        required=True,
                        help="The name of the task to train.")
    parser.add_argument("--output_dir",
                        default=None,
                        type=str,
                        required=True,
                        help="The output directory where the model predictions and checkpoints will be written.")
    parser.add_argument("--cache_dir",
                        default="",
                        type=str,
                        help="Where do you want to store the pre-trained models downloaded from s3")
    parser.add_argument("--max_seq_length",
                        default=128,
                        type=int,
                        help="The maximum total input sequence length after WordPiece tokenization. \n"
                             "Sequences longer than this will be truncated, and sequences shorter \n"
                             "than this will be padded.")
    parser.add_argument("--do_eval",
                        action='store_true',
                        help="Whether to run eval on the dev set.")
    parser.add_argument("--do_lower_case",
                        action='store_true',
                        help="Set this flag if you are using an uncased model.")
    parser.add_argument("--train_batch_size",
                        default=32,
                        type=int,
                        help="Total batch size for training.")
    parser.add_argument("--eval_batch_size",
                        default=32,
                        type=int,
                        help="Total batch size for eval.")
    parser.add_argument("--learning_rate",
                        default=5e-5,
                        type=float,
                        help="The initial learning rate for Adam.")
    parser.add_argument('--weight_decay', '--wd',
                        default=1e-4,
                        type=float,
                        metavar='W',
                        help='weight decay')
    parser.add_argument("--num_train_epochs",
                        default=3.0,
                        type=float,
                        help="Total number of training epochs to perform.")
    parser.add_argument("--warmup_proportion",
                        default=0.1,
                        type=float,
                        help="Proportion of training to perform linear learning rate warmup for. "
                             "E.g., 0.1 = 10%% of training.")
    parser.add_argument("--no_cuda",
                        action='store_true',
                        help="Whether not to use CUDA when available")
    parser.add_argument('--seed',
                        type=int,
                        default=42,
                        help="random seed for initialization")
    parser.add_argument('--gradient_accumulation_steps',
                        type=int,
                        default=1,
                        help="Number of updates steps to accumulate before performing a backward/update pass.")

    # added arguments
    parser.add_argument('--aug_train',
                        action='store_true')
    parser.add_argument('--eval_step',
                        type=int,
                        default=50)
    parser.add_argument('--data_url',
                        type=str,
                        default="")
    parser.add_argument('--temperature',
                        type=float,
                        default=1.)
    parser.add_argument('--embedding_size',
                        type=int,
                        default=300)
    parser.add_argument('--hidden_size4lstm',
                        type=float,
                        default=600)

    args = parser.parse_args()
    logger.info('The args: {}'.format(args))

    processors = {
        "sst-2": Sst2Processor,
    }

    output_modes = {
        "sst-2": "classification",
    }

    # intermediate distillation default parameters
    default_params = {
        "sst-2": {"num_train_epochs": 10, "max_seq_length": 64},
    }

    acc_tasks = ["mnli", "mrpc", "sst-2", "qqp", "qnli", "rte"]
    corr_tasks = ["sts-b"]
    mcc_tasks = ["cola"]

    # Prepare devices
    device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
    n_gpu = torch.cuda.device_count()

    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
                        datefmt='%m/%d/%Y %H:%M:%S',
                        level=logging.INFO)

    logger.info("device: {} n_gpu: {}".format(device, n_gpu))

    # Prepare seed
    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    if n_gpu > 0:
        torch.cuda.manual_seed_all(args.seed)

    # Prepare task settings
    if os.path.exists(args.output_dir) and os.listdir(args.output_dir):
        raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir))
    if not os.path.exists(args.output_dir):
        os.makedirs(args.output_dir)

    task_name = args.task_name.lower()

    if task_name in default_params:
        args.max_seq_len = default_params[task_name]["max_seq_length"]

    if not args.do_eval:
        if task_name in default_params:
            args.num_train_epoch = default_params[task_name]["num_train_epochs"]

    if task_name not in processors:
        raise ValueError("Task not found: %s" % task_name)

    processor = processors[task_name]()
    output_mode = output_modes[task_name]
    label_list = processor.get_labels()
    num_labels = len(label_list)

    tokenizer = BiLstmTokenizer(vocab_file=args.glove_vocab_file)

    if not args.do_eval:
        if not args.aug_train:
            train_examples = processor.get_train_examples(args.data_dir)
        else:
            train_examples = processor.get_aug_examples(args.data_dir)
        if args.gradient_accumulation_steps < 1:
            raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format(
                args.gradient_accumulation_steps))

        args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps

        num_train_optimization_steps = int(
            len(train_examples) / args.train_batch_size / args.gradient_accumulation_steps) * args.num_train_epochs

        # 这里需要生成两种 train_dataloader
        train_features = convert_examples_to_features4lstm(train_examples, label_list,
                                                                args.max_seq_length, tokenizer, output_mode)
        train_data, _ = get_tensor_data(output_mode, train_features)
        train_sampler = RandomSampler(train_data)
        train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size)

    eval_examples = processor.get_dev_examples(args.data_dir)
    eval_features = convert_examples_to_features4lstm(eval_examples, label_list, args.max_seq_length, tokenizer, output_mode)
    eval_data, eval_labels = get_tensor_data(output_mode, eval_features)
    eval_sampler = SequentialSampler(eval_data)
    eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size)

    lstm_model = BiLSTM(embedding_size=args.embedding_size, n_hidden1=args.hidden_size4lstm, n_class=num_labels, max_len=args.max_seq_len)
    lstm_model.to(device)

    if args.do_eval:
        logger.info("***** Running evaluation *****")
        logger.info("  Num examples = %d", len(eval_examples))
        logger.info("  Batch size = %d", args.eval_batch_size)

        lstm_model.eval()
        result = do_eval(lstm_model, task_name, eval_dataloader,
                         device, output_mode, eval_labels, num_labels)
        logger.info("***** Eval results *****")
        for key in sorted(result.keys()):
            logger.info("  %s = %s", key, str(result[key]))
    else:
        logger.info("***** Running training *****")
        logger.info("  Num examples = %d", len(train_examples))
        logger.info("  Batch size = %d", args.train_batch_size)
        logger.info("  Num steps = %d", num_train_optimization_steps)
        if n_gpu > 1:
            lstm_model = torch.nn.DataParallel(lstm_model)
        # Prepare optimizer
        param_optimizer = list(lstm_model.named_parameters())
        size = 0
        for n, p in lstm_model.named_parameters():
            logger.info('n: {}'.format(n))
            size += p.nelement()

        logger.info('Total parameters: {}'.format(size))
        no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
        optimizer_grouped_parameters = [
            {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
            {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
        ]
        # schedule = 'none'
        # optimizer = BertAdam(optimizer_grouped_parameters,
        #                      schedule=schedule,
        #                      lr=args.learning_rate,
        #                      warmup=args.warmup_proportion,
        #                      t_total=num_train_optimization_steps)
        # Prepare loss functions
        optimizer = torch.optim.adadelta.Adadelta(optimizer_grouped_parameters)
        loss_mse = MSELoss()

        def soft_cross_entropy(predicts, targets):
            student_likelihood = torch.nn.functional.log_softmax(predicts, dim=-1)
            targets_prob = torch.nn.functional.softmax(targets, dim=-1)
            return (- targets_prob * student_likelihood).mean()

        # Train and evaluate
        global_step = 0
        best_dev_acc = 0.0
        output_eval_file = os.path.join(args.output_dir, "eval_results.txt")

        for epoch_ in trange(int(args.num_train_epochs), desc="Epoch"):
            tr_loss = 0.

            lstm_model.train()
            nb_tr_examples, nb_tr_steps = 0, 0

            for step, batch in enumerate(tqdm(train_dataloader, desc="Iteration", ascii=True)):
                batch = tuple(t.to(device) for t in batch)

                input_ids, input_mask, segment_ids, label_ids, seq_lengths = batch
                if input_ids.size()[0] != args.train_batch_size:
                    continue

                input_ids, _, _, _, seq_lengths = batch
                _, _, logits, reps = lstm_model(input_ids, seq_lengths)

                loss_fct = CrossEntropyLoss()
                loss = loss_fct(logits.view(-1, num_labels), label_ids.view(-1))

                if n_gpu > 1:
                    loss = loss.mean()  # mean() to average on multi-gpu.
                if args.gradient_accumulation_steps > 1:
                    loss = loss / args.gradient_accumulation_steps

                loss.backward()

                tr_loss += loss.item()
                nb_tr_examples += label_ids.size(0)
                nb_tr_steps += 1

                if (step + 1) % args.gradient_accumulation_steps == 0:
                    optimizer.step()
                    optimizer.zero_grad()
                    global_step += 1

                if (global_step + 1) % args.eval_step == 0:
                    logger.info("***** Running evaluation *****")
                    logger.info("  Epoch = {} iter {} step".format(epoch_, global_step))
                    logger.info("  Num examples = %d", len(eval_examples))
                    logger.info("  Batch size = %d", args.eval_batch_size)

                    lstm_model.eval()

                    loss = tr_loss / (step + 1)

                    result = do_eval(lstm_model, task_name, eval_dataloader,
                                     device, output_mode, eval_labels, num_labels)
                    result['global_step'] = global_step
                    result['loss'] = loss

                    result_to_file(result, output_eval_file)

                    save_model = False

                    if task_name in acc_tasks and result['acc'] > best_dev_acc:
                        best_dev_acc = result['acc']
                        save_model = True

                    if task_name in corr_tasks and result['corr'] > best_dev_acc:
                        best_dev_acc = result['corr']
                        save_model = True

                    if task_name in mcc_tasks and result['mcc'] > best_dev_acc:
                        best_dev_acc = result['mcc']
                        save_model = True

                    if save_model:
                        logger.info("***** Save model *****")
                        model_to_save = lstm_model.module if hasattr(lstm_model, 'module') else lstm_model
                        model_name = WEIGHTS_NAME
                        output_model_file = os.path.join(args.output_dir, model_name)
                        torch.save(model_to_save, output_model_file)

                    lstm_model.train()

if __name__ == "__main__":
    main()


from functools import reduce

lst = [5,7,3,43,54,6,76,12]

# Normal Way
# def list_sum(data):
#     sum = 0
#     for i in data:
#         sum+=i
#     return sum
# result = list_sum(lst)
# print("Result : ",result)

# Using Reduce
# def list_sum(data,data1):
#     return data + data1

# result = reduce(list_sum,lst)
# print("Result : ",result)

# Using Lambda and Reduce
result=reduce(lambda data,data1:data+data1,lst)
print("sum is:",result)


# Windows badge flashing utility, submitted by ianjfrosst
import serial
import sys

# Set this to your serial adapter
PORT = 'COM0'

def load(fname):
    with open(fname, 'rb') as f:
        pgm = f.read()

    with serial.Serial(PORT) as s:
        s.write(pgm)
        s.flush()
        s.write(b'\0' * 1024) # sometimes flush doesn't seem to work. this'll fix it

if len(sys.argv) <= 1:
    print("Must provide .hex file to flash!")
    exit()

load(sys.argv[1])


"""Main."""

# Utilities
from typing import List

# FastApi
from fastapi import FastAPI
from fastapi.params import Depends

# SQLalchemy
from sqlalchemy.orm.session import Session

# Starlette
from starlette import status

# Database
from app import services

# Models
from app.models import Post

# Schemas
from app.schemas import PostSchema

app = FastAPI()

services.create_database()

@app.get(path='/', status_code=status.HTTP_200_OK)
def home():
    return {'Saludo': 'Hello World!'}

@app.get(
    path='/posts/',
    response_model=List[PostSchema],
    status_code=status.HTTP_200_OK
)
def list_posts(db: Session=Depends(services.get_db)):
    """List all posts."""
    posts = db.query(Post).all()
    return posts

@app.get(
    path='/posts/{id}',
    response_model=PostSchema,
    status_code=status.HTTP_200_OK
)
def get_post(id: int, db: Session=Depends(services.get_db)):
    """Retrive a post."""
    post = db.query(Post).filter(Post.id==id)


import pytest

import pandas as pd

import sys, os
home_directory = '/home/matthew'
sys.path.append(home_directory+'/food-diary-analysis/lib')
print(sys.path)
from pipeline_components import pipeline_components as pc

def run_component(df, options={}):
    return pc.ConvertStringColumnToListOfStrings(df, options).run()

def test_empty_dataframe():
    input_df = pd.DataFrame({})

    with pytest.raises(ValueError) as e:
        run_component(input_df)

    assert str(e.value) == 'Some or all required columns [\'\'] are not found in input dataframe columns []'

def test_dataframe_with_missing_required_column():
    input_df = pd.DataFrame(
        {
            'col1': [None],
            'col2': [None],
        }
    )

    with pytest.raises(ValueError) as e:
        run_component(input_df, {"column_to_convert": "not_a_col"})

    assert str(e.value) == 'Some or all required columns [\'not_a_col\'] are not found in input dataframe columns [\'col1\', \'col2\']'

def test_dataframe_with_default_required_column():
    input_df = pd.DataFrame(
        {
            'col1': [None],
            'col2': [None],
        }
    )

    with pytest.raises(ValueError) as e:
        run_component(input_df)

    assert str(e.value) == 'Some or all required columns [\'\'] are not found in input dataframe columns [\'col1\', \'col2\']'

def test_wrong_dtype_of_column():
    input_df = pd.DataFrame(
        {
            'col': [4]
        }
    )

    with pytest.raises(ValueError) as e:
        run_component(input_df, {"column_to_convert": "col"})

    assert str(e.value) == 'Column to convert col is not of dtype Object, but is of type int64'

def test_input_dataframe_with_default_separater():
    input_df = pd.DataFrame(
        {
            'col': ['this, is, a, bunch, of, strings', 'another, bunch, of, strings']
        }
    )

    result_df = run_component(input_df, {"column_to_convert": "col"})

    expected_df = pd.DataFrame(
        {
            'col': [['this', 'is', 'a', 'bunch', 'of', 'strings'], ['another', 'bunch', 'of', 'strings']]
        }
    )

    pd.testing.assert_frame_equal(result_df, expected_df)


import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
# Імпорт Axes3D має побічні ефекти, він дозволяє використовувати projection = '3d' у add_subplot
from tkinter import *
import numpy as np
from numpy import sin, cos
import matplotlib.animation as animation
import time

def figura():
    l = 4  # довжина (length)
    w = 3  # ширина (width)
    h = 5  # висота (height)
    color_prism = "black"  # колір призми
    lw = 0.5  # товщина граней
    # задаємо відрізки для призми
    x = [[0, 0], [0, 0], [l, l], [l, l], [0, 0], [0, 0], [l, l], [l, l], [0, l], [0, l], [0, l], [0, l]]
    y = [[0, 0], [w, w], [0, 0], [w, w], [0, w], [0, w], [0, w], [0, w], [0, 0], [0, 0], [w, w], [w, w]]
    z = [[0, h], [0, h], [0, h], [0, h], [h, h], [0, 0], [h, h], [0, 0], [0, 0], [h, h], [0, 0], [h, h]]

    fig = plt.figure()
    ax = fig.add_subplot(projection='3d')
    ax.set()  # facecolor="w")
    fig.set()  # facecolor="w")

    def ox(cant_x):
        for i in range(12):
            aa = [x[i][0], y[i][0] * cos(cant_x) - z[i][0] * sin(cant_x), y[i][0] * sin(cant_x) + z[i][0] * cos(cant_x)]
            bb = [x[i][1], y[i][1] * cos(cant_x) - z[i][1] * sin(cant_x), y[i][1] * sin(cant_x) + z[i][1] * cos(cant_x)]
            ax.plot(*zip(aa, bb), color=color_prism, linewidth=lw)

    def oy(cant_y):
        for i in range(12):
            aa = [x[i][0] * cos(cant_y) + z[i][0] * sin(cant_y), y[i][0],
                  -x[i][0] * sin(cant_y) + z[i][0] * cos(cant_y)]
            bb = [x[i][1] * cos(cant_y) + z[i][1] * sin(cant_y), y[i][1],
                  -x[i][1] * sin(cant_y) + z[i][1] * cos(cant_y)]
            ax.plot(*zip(aa, bb), color=color_prism, linewidth=lw)

    def oz(cant_z):
        for i in range(12):
            aa = [x[i][0] * cos(cant_z) - y[i][0] * sin(cant_z), x[i][0] * sin(cant_z) + y[i][0] * cos(cant_z), z[i][0]]
            bb = [x[i][1] * cos(cant_z) - y[i][1] * sin(cant_z), x[i][1] * sin(cant_z) + y[i][1] * cos(cant_z), z[i][1]]
            ax.plot(*zip(aa, bb), color=color_prism, linewidth=lw)

    def oxy(cant_xy):
        for i in range(12):
            aa = [x[i][0], y[i][0] * cos(cant_xy) - z[i][0] * sin(cant_xy),
                  y[i][0] * sin(cant_xy) + z[i][0] * cos(cant_xy)]
            bb = [x[i][1], y[i][1] * cos(cant_xy) - z[i][1] * sin(cant_xy),
                  y[i][1] * sin(cant_xy) + z[i][1] * cos(cant_xy)]
            s = [aa[0] * cos(cant_xy) + aa[2] * sin(cant_xy), aa[1], -aa[0] * sin(cant_xy) + aa[2] * cos(cant_xy)]
            e = [bb[0] * cos(cant_xy) + bb[2] * sin(cant_xy), bb[1], -bb[0] * sin(cant_xy) + bb[2] * cos(cant_xy)]
            ax.plot(*zip(s, e), color=color_prism, linewidth=lw)

    def change(cant):
        if var.get() == 0:
            ox(cant)
        elif var.get() == 1:
            oy(cant)
        elif var.get() == 2:
            oz(cant)
        elif var.get() == 3:
            oxy(cant)

    def animate(i):
        ax.clear()
        m_xyz = max(l, w, h)
        ax.set_xlim(-m_xyz, m_xyz)
        ax.set_ylim(-m_xyz, m_xyz)
        ax.set_zlim(-m_xyz, m_xyz)
        ax.set_xlabel('X Label')
        ax.set_ylabel('Y Label')
        ax.set_zlabel('Z Label')
        cant = np.radians(i)
        change(cant)

    run_animation = animation.FuncAnimation(fig, animate, interval=10000 / speed.get())

    plt.show()

def gui():
    global var, speed
    root = Tk()
    root.geometry('300x200+1100+100')
    root.title("TeriatkinOnishuk")
    var = IntVar()
    var.set(0)
    speed = IntVar()
    speed.set(100)
    x_button = Radiobutton(text="X", variable=var, value=0)
    y_button = Radiobutton(text="Y", variable=var, value=1)
    z_button = Radiobutton(text="Z", variable=var, value=2)
    xy_button = Radiobutton(text="XY", variable=var, value=3)
    sp_label = Label(text="Speed\n[1 - 1000]")
    sp_entry = Entry(textvariable=speed)
    button = Button(text="RUN animations", command=figura)
    label = Label(width=20, height=10)
    x_button.pack()
    y_button.pack()
    z_button.pack()
    xy_button.pack()
    sp_label.pack()
    sp_entry.pack()
    button.pack()
    label.pack()
    root.mainloop()

gui()


import pandas as pd

friend_dict_list = [
    {"name": "John", "age": 25, "job": "student"},
    {"name": "Nate", "age": 30, "job": "teacher"}
]

df = pd.DataFrame(friend_dict_list) #데이터프레임 생성
df = df[["name","age","job"]] # 컬럼이 입력한 순서대로 정렬이 안되서 재정렬 필요
print(df.head())
print()

###입력 순서가 보장되게 하자###
from collections import OrderedDict

friend_ordered_dict = OrderedDict(
    [
        ("name",["John","Nate"]),
        ("age",[25,30]),
        ("job",["student", "teacher"])
    ]
)

df2 = pd.DataFrame(friend_ordered_dict) #데이터프레임 생성, 입력순서 보장
print(df2.head())
print()

friend_list = [
    ["John", 25, "student"],
    ["Nate", 30, "teacher"]
]
column_name = ["name", "age", "job"]
df3 = pd.DataFrame(friend_list,columns=column_name,index=list(range(len(friend_list)))) #컬림이름, 인덱스 넣어줌
print(df3.head())


import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

setuptools.setup(
    name="nlpt5",
    version="1.0.0",
    author="adamnpeace",
    author_email="test@email.com",
    description="Pipelines for T5",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/adamnpeace/nlpt5",
    packages=setuptools.find_packages(),
    classifiers=(
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ),
    install_requires=[
        'nlp==0.2.0',
        'datasets==1.6.2',
        'transformers==3.0.0',
        'torch>=1.8.0'
    ]
)