import sys
import math
import random
import numpy as np
from tabulate import tabulate
from data_augmentation import jittering, smote_based_wDBA, window_slicing
from enum import Enum
from utility.conversion_methods import join_dataset_dictionaries, fix_rotation_matrices, list_to_matrix

PERFORMANCE_MEASURES = ["Binary crossentropy loss", "Binary accuracy", "AUC", "f1-score", "Precision", "Recall"]
DATA_AUGMENTATION_TECHNIQUES = ["smote_based_wdba", "jittering"]

class ProficiencyLabel(Enum):
    """
        Scanned region
    """
    Novice = 0
    Expert = 1

# Represents the data for a single surgery. Stores time series data captured from the surgery, as well as related info
# May store multiple time series if multiple sequence types are considered (assumption is they all share the same timestamps)

class SurgeryData:

    def __init__(self, skill_level, surgery_type, timestamps=[]):
        self.sequences = dict()
        self.skill_level = skill_level
        self.surgery_type = surgery_type
        self.timestamps = timestamps

    def add_sequence(self, sequence_type, time_series):
        self.sequences[sequence_type] = time_series

# Instances stores all surgeries data recorded for some participant
class ParticipantData:

    def __init__(self, name):
        self.name = name
        self.surgeries = []
        self.surgeries_stats = dict(
            novices_IP=0,
            novices_OOP=0,
            experts_IP=0,
            experts_OOP=0
        )

    # To add a surgery, must pass in a Surgery object
    def add_surgery(self, surgery):

        self.surgeries.append(surgery)
        self.update_surgery_stats(surgery)

    def update_surgery_stats(self, surgery):

        skill_level = surgery.skill_level
        surgery_type = surgery.surgery_type
        for stat in self.surgeries_stats:
            if (skill_level.lower() in stat.lower()) and (surgery_type.lower() in stat.lower()):
                self.surgeries_stats[stat] += 1
                break

# Contains a dictionary that stores participant names as keys, and the corresponding ParticipantsStorage objects as keys
# Altogether, it contains all time series data
class ParticipantsStorage:

    def __init__(self):
        self.participants = dict()
        self.surgeries_stats = dict(
            surgeries=0,
            novices=0,
            experts=0,
            novices_IP=0,
            novices_OOP=0,
            experts_IP=0,
            experts_OOP=0
        )

    # Adds a surgery to the storage, within the corresponding participant's storage. If the participant does not exist,
    # then they are created
    def add_surgery(self, participant_name, surgery):

        if participant_name not in self.participants:
            self.participants[participant_name] = ParticipantData(participant_name)
        self.participants[participant_name].add_surgery(surgery)
        self.update_surgeries_stats(surgery)

    def add_participant(self, participant):

        if (not isinstance(participant, ParticipantData)) or participant.name in self.participants:
            return False
        self.participants[participant.name] = participant
        for surgery in participant.surgeries:
            self.update_surgeries_stats(surgery)
        return True

    def pop_participant(self, participant_name=None):

        if len(self.participants) == 0:
            return None
        participant = None
        names = list(self.participants.keys())
        if participant_name is None:
            # Pops random participant
            participant = self.participants.pop(names[random.randint(0, len(self.participants)-1)])
        elif participant_name in names:
            participant = self.participants.pop(participant_name)
        else:
            return None

        for surgery in participant.surgeries:
            self.update_surgeries_stats(surgery, False)

        return participant

    def update_surgeries_stats(self, surgery, add=True):

        # Values we add below depend on whether a surgery is being added or removed
        term = 1 if add else -1
        skill_level = surgery.skill_level
        surgery_type = surgery.surgery_type
        self.surgeries_stats["surgeries"] += term
        self.surgeries_stats[(skill_level + "s").lower()] += term  # Updates number of novices / experts
        for stat in self.surgeries_stats:
            if (skill_level.lower() in stat.lower()) and (surgery_type.lower() in stat.lower()):
                self.surgeries_stats[stat] += term
                break

    # Note: In all ratios, we return math.inf if denominator is 0
    def novices_to_experts_ratio(self):

        return self.surgeries_stats["novices"] / self.surgeries_stats["experts"] if  \
            self.surgeries_stats["experts"] else math.inf

    def novices_IP_to_OOP_ratio(self):

        return self.surgeries_stats["novices_IP"] / self.surgeries_stats["novices_OOP"] if \
            self.surgeries_stats["novices_OOP"] else math.inf

    def experts_IP_to_OOP_ratio(self):

        return self.surgeries_stats["experts_IP"] / self.surgeries_stats["experts_OOP"] if \
            self.surgeries_stats["experts_OOP"] else 0

class Jittering:

    def __init__(self, rotation_magnitude=1, translation_magnitude=1):

        self.rotation_magnitude = rotation_magnitude
        self.translation_magnitude = translation_magnitude

    def execute(self, dataset, synthetic_amount):
        return jittering.jittering_specified_amount(
            dataset, synthetic_amount, self.rotation_magnitude, self.translation_magnitude)

class SmoteBasedWDBA:

    def __init__(self, k=None):
        self.k = k

    def execute(self, dataset, synthetic_amount):
        return smote_based_wDBA.smote_based_weighted_dba_specified_amount(
            dataset, synthetic_amount, self.k)

class WindowSlicing:

    def __init__(self, window_slice_prop=0.9):
        self.window_slice_prop = window_slice_prop

    def execute(self, dataset, synthetic_amount):
        return window_slicing.window_slicing_specified_amount(dataset, synthetic_amount, self.window_slice_prop)

class DataAugmentationInstruction:

    # Technique refers to the data augmentation technique to use
    def __init__(self, technique, augment_synthetic=False):
        self.technique = technique
        self.augment_synthetic = augment_synthetic

class BalanceDataset(DataAugmentationInstruction):

    def __init__(self, technique, augment_synthetic=False):

        super().__init__(technique, augment_synthetic)

    # Here is the code for balancing a dataset. First balance novices IP, OOP and experts IP, OOP.
    # Afterwards, balance novices and experts . We always up-sample, via the specified technique.
    # Returns synthetic samples
    def execute(self, original_dataset, synthetic_dataset):

        enhanced_set = join_dataset_dictionaries(original_dataset, synthetic_dataset)
        # print("Num of novices IP:", len(enhanced_set["Novices"]["IP"]))
        # print("Num of novices OOP:", len(enhanced_set["Novices"]["OOP"]))
        # print("Num of experts OP:", len(enhanced_set["Experts"]["IP"]))
        # print("Num of experts OOP:", len(enhanced_set["Experts"]["OOP"]))

        # If enhanced_set assigned, then new samples may be creating using the synthetic ones passed in.
        set_used_for_augmentation = enhanced_set if self.augment_synthetic else original_dataset

        # Balance novices IP and OOP, and then experts IP, OOP (in enhanced set)
        for skill_level in ["Novices", "Experts"]:
            desired_size = max(len(enhanced_set[skill_level]["IP"]), len(enhanced_set[skill_level]["OOP"]))
            for surgery_type in ["IP", "OOP"]:
                synthetic_amount = desired_size - len(enhanced_set[skill_level][surgery_type])
                synthetic_dataset[skill_level][surgery_type] += \
                    self.technique.execute(set_used_for_augmentation[skill_level][surgery_type], synthetic_amount)

        # Balance novices and experts. At this point, 1:1 IP to OOP ratio for both skill levels
        num_novices_IP = len(original_dataset["Novices"]["IP"]) + len(synthetic_dataset["Novices"]["IP"])
        num_experts_IP = len(original_dataset["Experts"]["IP"]) + len(synthetic_dataset["Experts"]["IP"])
        desired_size = max(num_novices_IP, num_experts_IP)

        # print("num_novices_IP:", num_novices_IP)
        # print("num_experts_IP", num_experts_IP)

        for skill_level in ["Novices", "Experts"]:
            for surgery_type in ["IP", "OOP"]:
                # Want to get to the desired size. Need to account both original and synthetic samples
                synthetic_amount = desired_size - (len(original_dataset[skill_level][surgery_type]) +
                                                   len(synthetic_dataset[skill_level][surgery_type]))
                synthetic_dataset[skill_level][surgery_type] += \
                    self.technique.execute(set_used_for_augmentation[skill_level][surgery_type], synthetic_amount)

        return synthetic_dataset

# Novices IP, Novices OOP, Experts IP, Experts OOP, which make up the dataset, are all increased by the given factor
# independently.
# NOTE: ** Increase is relative to the size of the enhanced set, not the original dataset
class IncreaseDatasetProportionally(DataAugmentationInstruction):

    def __init__(self, technique, increase_factor, augment_synthetic=False):

        super().__init__(technique, augment_synthetic)
        self.increase_factor = increase_factor

    def execute(self, original_dataset, synthetic_dataset):

        enhanced_set = join_dataset_dictionaries(original_dataset, synthetic_dataset)
        set_used_for_augmentation = enhanced_set if self.augment_synthetic else original_dataset

        if self.increase_factor <= 1:
            return synthetic_dataset

        # Increase each division separately
        for skill_level in ["Novices", "Experts"]:
            for surgery_type in ["IP", "OOP"]:
                desired_size = round(self.increase_factor * len(enhanced_set[skill_level][surgery_type]))
                synthetic_amount = desired_size - len(enhanced_set[skill_level][surgery_type])
                synthetic_dataset[skill_level][surgery_type] += \
                    self.technique.execute(set_used_for_augmentation[skill_level][surgery_type], synthetic_amount)

        return synthetic_dataset

# Novices IP, Novices OOP, Experts IP, Experts OOP, which make up the dataset, are all increased by aprx. the same number
# of synthetic samples such that the dataset reaches the required size, which is len(original_dataset) * increase_factor.
# -This means increase is relative to size of the original dataset
# NOTE: the resulting enhanced set may be slightly imbalanced
#  e.g perhaps 4 Novices IP are required and 3 for each of the other types
# ** NOTE: This method only works well for increase_factor >= 2 . If it's less, increase_factor may already be exceeded
# when samples are first balanced

class IncreaseDatasetBalanced(DataAugmentationInstruction):

    def __init__(self, technique, increase_factor, augment_synthetic=False):

        super().__init__(technique, augment_synthetic)
        self.increase_factor = increase_factor
        self.balance_instruction = BalanceDataset(technique, augment_synthetic)

    def execute(self, original_dataset, synthetic_dataset):

        if self.increase_factor < 2:
            print("Note: Increase factor must be >= 2. No data augmentation performed... (otherwise factor is overshot when balancing)")
            return synthetic_dataset

        # First balance dataset
        synthetic_dataset = self.balance_instruction.execute(original_dataset, synthetic_dataset)

        enhanced_set = join_dataset_dictionaries(original_dataset, synthetic_dataset)
        set_used_for_augmentation = enhanced_set if self.augment_synthetic else original_dataset

        # Find number of surgeries in original and enhanced datasets
        original_size = 0
        cur_size = 0
        for skill_level in ["Novices", "Experts"]:
            for surgery_type in ["IP", "OOP"]:
                original_size += len(original_dataset[skill_level][surgery_type])
                cur_size += len(enhanced_set[skill_level][surgery_type])

        desired_size = round(self.increase_factor * original_size)

        synthetic_samples_required = desired_size - cur_size

        if synthetic_samples_required <= 0:
            return synthetic_dataset

        # Increase each division separately, all by aprx. the same amount
        # After attempting to split data augmentation evenly, may require 1 more of a certain division(s)
        # then others. This occurs if desired_size is not a multiple of 4
        extra_samples_required = synthetic_samples_required % 4
        for skill_level in ["Novices", "Experts"]:
            for surgery_type in ["IP", "OOP"]:
                synthetic_amount = synthetic_samples_required // 4
                if extra_samples_required > 0:
                    synthetic_amount += 1
                    extra_samples_required -= 1

                synthetic_dataset[skill_level][surgery_type] += \
                    self.technique.execute(set_used_for_augmentation[skill_level][surgery_type], synthetic_amount)

        return synthetic_dataset

class DataAugmentationController:

    # Instructions stores list of data augmentation steps to perform
    def __init__(self, instructions=None):
        if instructions is None:
            self.instructions = []
        else:
            self.instructions = instructions

    def add_instruction(self, instruction):
        self.instructions.append(instruction)

    def execute(self, original_dataset):
        synthetic_dataset = dict(
            Novices=dict(IP=[], OOP=[]),
            Experts=dict(IP=[], OOP=[])
        )
        for instruction in self.instructions:
            synthetic_dataset = instruction.execute(original_dataset, synthetic_dataset)

        # print("Matrix multiplication before:")
        # product = np.dot(list_to_matrix(synthetic_dataset["Novices"]["IP"][0][0])[0:3, 0:3],
        #                  list_to_matrix(synthetic_dataset["Novices"]["IP"][0][0])[0:3, 0:3].transpose())
        # np.savetxt(sys.stdout, product, '%.5f')

        # Fix all rotation matrices, for all time series
        for skill_level in synthetic_dataset:
            for surgery_type in synthetic_dataset[skill_level]:
                for i in range(len(synthetic_dataset[skill_level][surgery_type])):
                    fix_rotation_matrices(synthetic_dataset[skill_level][surgery_type][i])

        # print("Matrix multiplication after:")
        # product = np.dot(list_to_matrix(synthetic_dataset["Novices"]["IP"][0][0])[0:3, 0:3],
        #                  list_to_matrix(synthetic_dataset["Novices"]["IP"][0][0])[0:3, 0:3].transpose())
        # np.savetxt(sys.stdout, product, '%.5f')

        return synthetic_dataset

class Results:

    def __init__(self):

        self.averages = dict(
            training=dict(),
            validation=dict(),
            test=dict()
        )
        self.stddevs = dict(
            training=dict(),
            validation=dict(),
            test=dict()
        )

    def __str__(self):

        # Averages
        lists_to_print = []
        for performance_measure in PERFORMANCE_MEASURES:
            lists_to_print.append([performance_measure, self.averages["training"][performance_measure],
                                   self.averages["validation"][performance_measure],
                                   self.averages["test"][performance_measure]])

        averages_table = tabulate(tabular_data=lists_to_print,
                                  headers=["Performance measure", "All training results - mean",
                                           "All validation results - mean", "Tests - mean"])
        # Stddevs
        lists_to_print = []
        for performance_measure in PERFORMANCE_MEASURES:
            lists_to_print.append([performance_measure, self.stddevs["training"][performance_measure],
                                   self.stddevs["validation"][performance_measure],
                                   self.stddevs["test"][performance_measure]])

        stddevs_table = tabulate(tabular_data=lists_to_print,
                                 headers=["Performance measure", "All training results - stddev",
                                          "All validation results - stddev", "Tests - stddev"])

        return averages_table + "\n\n" + stddevs_table + "\n"

    @staticmethod
    def create_averaged_results_object(all_result_objects):

        # Initializing averaged results object
        averaged_results = Results()
        for set in averaged_results.averages:
            for performance_measure in PERFORMANCE_MEASURES:
                averaged_results.averages[set][performance_measure] = 0
                averaged_results.stddevs[set][performance_measure] = 0

        for single_results in all_result_objects:
            for set in averaged_results.averages:
                for performance_measure in PERFORMANCE_MEASURES:
                    averaged_results.averages[set][performance_measure] += \
                        single_results.averages[set][performance_measure] / len(all_result_objects)
                    averaged_results.stddevs[set][performance_measure] += \
                        single_results.stddevs[set][performance_measure] / len(all_result_objects)
        return averaged_results

class Transcript(object):
    """
    Transcript - direct print output to a file, in addition to terminal (from Stack Overflow, but modified).

    Usage:
        import transcript
        transcript.start('logfile.log')
        print("inside file")
        transcript.stop()
        print("outside file")
    """

    def __init__(self, filename):
        self.terminal = sys.stdout
        self.logfile = open(filename, "a")

    def write(self, message):

        self.terminal.write(message)
        self.logfile.write(message)

    def flush(self):
        # this flush method is needed for python 3 compatibility.
        # this handles the flush command by doing nothing.
        # you might want to specify some extra behavior here.
        pass


import collections
import sys

import pytest

from extendable import context, main, registry

@pytest.fixture
def test_registry() -> registry.ExtendableClassesRegistry:
    reg = registry.ExtendableClassesRegistry()
    initial_class_defs = main._extendable_class_defs_by_module
    try:
        main._extendable_class_defs_by_module = collections.OrderedDict()
        token = context.extendable_registry.set(reg)
        yield reg
    finally:
        main._extendable_class_defs_by_module = initial_class_defs
        context.extendable_registry.reset(token)

@pytest.fixture
def sys_modules_cleanup() -> None:
    keys = [k for k in sys.modules.keys() if k.startswith("tests.mod_")]
    for k in keys:
        del sys.modules[k]
    try:
        yield
    finally:
        keys = [k for k in sys.modules.keys() if k.startswith("tests.mod_")]
        for k in keys:
            del sys.modules[k]


from rest_framework import serializers
from .models import *
from  django.contrib.auth.models import User

class Userserializer(serializers.ModelSerializer):
    class Meta:
        model= User
        fields=('id','username','email')

class Registrationserializer(serializers.ModelSerializer):
    class Meta:
        model= User
        fields=('id','username','email','password')
        extra_kwargs={'password':{'write_only':True}}
    def create(self, validated_data):
        user= User.objects.create_user(validated_data['username'],validated_data['email'],validated_data['password'])
        return user
class Userprofileserializer(serializers.ModelSerializer):
    #user= User.objects.create_user(validated_data['username'],validated_data['email],validated_data['password]);
    class Meta:
        model=User
        fields=['id','username','email']

class categoryserializer(serializers.ModelSerializer):
    class Meta:
          model= Category
          fields=(
            'id',
            'title',
            'image'
          )
class Authorserializer(serializers.ModelSerializer):
    class Meta:
        model=author
        fields=(
            'id',
            'name'

        )
class Publisherserializer(serializers.ModelSerializer):
    class Meta:
        model=publisher
        fields=(
            'id',
            'name'

        )
class Bookserializer(serializers.ModelSerializer):
    author_detail=Authorserializer(source='author',read_only=True,many=False)
    publisher_detail=Publisherserializer(source='publisher',read_only=True,many=False)
    category_detail=categoryserializer(source='category',read_only=True,many=False)
    class Meta:
        model= Books
        fields=(
           'id',
           'name',
           'category',
           'category_detail',
           'selling_price',
           'discounted_price',
           'author',
           'author_detail',
           'publisher',
           'publisher_detail',
           'image',
           'publisher'
)

class offerserializer(serializers.ModelSerializer):
    book_detail=Bookserializer(source='book',read_only=True,many=False)
    class Meta:
          model= Offer
          fields=(

            'id',
            'image',
            'book',
            'book_detail'
          )

class  customerserializer(serializers.ModelSerializer):
    class Meta:
        model=Customer
        fields=(
             'id',
             'user',
             'name',
             'locality',
             'city',
             'zipcode',
             'state'

        )

class update_item(serializers.ModelSerializer):
    class Meta:
        model=Cart
        fields=(
          'id',
          'quantity'

        )

class total(serializers.ModelSerializer):
    grand_total=serializers.SerializerMethodField(method_name='main_total')
    num_of_quantity=serializers.ReadOnlyField()
    num_of_book=serializers.ReadOnlyField()
    class Meta:
        model=Cart
        fields=[
            'num_of_quantity',
            'num_of_book',
            'grand_total',
        ]

    def main_total(self,request):
        user=request.user.id
        items=Cart.objects.filter(user=user)
        total=sum([item.quantity * item.book.discounted_price for item in items])
        return total

class cartserializers(serializers.ModelSerializer):
    #total_price=serializers.ReadOnlyField()
   # num_of_quantity=serializers.ReadOnlyField()
    book_detail=Bookserializer(source='book',read_only=True,many=False)
    price=serializers.ReadOnlyField()
   # total_price=serializers.SerializerMethodField(method_name='main_total')

    class Meta:
        model= Cart
        fields=(
            'id',
            'user',
            'book',
            'book_detail',
            'quantity',
            'price',

             )

    def validate(self, validated_data):
         user=validated_data["user"]
         books=validated_data["book"]
         current_item=  Cart.objects.filter(user=user,book=books)
         if not self.instance and current_item.count()>0:
             error={'Book already in your cart'}
             raise serializers.ValidationError(error)
         return  validated_data

class orderserializer(serializers.ModelSerializer):
    book_detail=Bookserializer(source='book',read_only=True,many=False)
    customer_detail=customerserializer(source='Customer',read_only=True,many=False)
    # cart_detail=cartserializers(source='cart',read_only=True,many=False)
    #book=Bookserializer()

    class Meta:
        model=orders
        fields=(
             'id',
             'user',
             'book',
             'book_detail',
             'customer',
             'quantity',
             'price',
             'customer_detail',
             'order_date',
             'status'
        )


#!/usr/bin/env python3
"""
does distributed blender rendering using NCS instances
"""

# standard library modules
import argparse
import contextlib
import getpass
import json
import logging
import math
import os
import socket
import shutil
import signal
import subprocess
import sys
import threading
import time
import uuid

# third-party module(s)
import requests

# neocortix modules
try:
    import ncs
except ImportError:
    # set system and python paths for default places, since path seems to be not set properly
    ncscliPath = os.path.expanduser('~/ncscli/ncscli')
    sys.path.append( ncscliPath )
    os.environ["PATH"] += os.pathsep + ncscliPath
    import ncs
import eventTiming
import jsonToKnownHosts
import purgeKnownHosts
import tellInstances

logger = logging.getLogger(__name__)

# possible place for globals is this class's attributes
class g_:
    signaled = False

class SigTerm(BaseException):
    #logger.warning( 'unsupported SigTerm exception created')
    pass

def sigtermHandler( sig, frame ):
    g_.signaled = True
    logger.warning( 'SIGTERM received; will try to shut down gracefully' )
    tellInstances.terminate()
    #raise SigTerm()

def sigtermSignaled():
    return g_.signaled

def boolArg( v ):
    '''use with ArgumentParser add_argument for (case-insensitive) boolean arg'''
    if v.lower() == 'true':
        return True
    elif v.lower() == 'false':
        return False
    else:
        raise argparse.ArgumentTypeError('Boolean value expected.')

def scriptDirPath():
    '''returns the absolute path to the directory containing this script'''
    return os.path.dirname(os.path.realpath(__file__))

def loadSshPubKey():
    pubKeyFilePath = os.path.expanduser( '~/.ssh/id_rsa.pub' )
    with open( pubKeyFilePath ) as inFile:
        contents = inFile.read()
    return contents

def launchInstances( authToken, nInstances, sshClientKeyName, launchedJsonFilepath,
        filtersJson=None, encryptFiles=True ):
    returnCode = 13
    # call ncs launch via command-line
    cmd = [
        'ncs.py', 'sc', '--authToken', authToken, 'launch',
        '--encryptFiles', str(encryptFiles),
        '--count', str(nInstances), # filtersArg,
        '--sshClientKeyName', sshClientKeyName, '--json'
    ]
    if filtersJson:
        cmd.extend( ['--filter',  filtersJson] )
    #logger.debug( 'cmd: %s', cmd )
    try:
        outFile = open( launchedJsonFilepath,'w' )
        #proc = subprocess.Popen( cmd, shell=True )
        proc = subprocess.Popen( cmd, stdout=outFile )
        while True:
            #logger.debug( 'polling ncs')
            proc.poll() # sets proc.returncode
            if proc.returncode != None:
                break
            if sigtermSignaled():
                logger.info( 'signaling ncs')
                proc.send_signal( signal.SIGTERM )
                try:
                    logger.info( 'waiting ncs')
                    proc.wait(timeout=60)
                    if proc.returncode:
                        logger.warning( 'ncs return code %d', proc.returncode )
                except subprocess.TimeoutExpired:
                    logger.warning( 'ncs launch did not terminate in time' )
            time.sleep( 1 )
        returnCode = proc.returncode
        if outFile:
            outFile.close()
    except Exception as exc:
        logger.error( 'exception while launching instances (%s) %s', type(exc), exc, exc_info=True )
        returnCode = 99
    return returnCode

def terminateThese( authToken, inRecs ):
    logger.info( 'to terminate %d instances', len(inRecs) )
    iids = [inRec['instanceId'] for inRec in inRecs]
    ncs.terminateInstances( authToken, iids )

def generateDtrConf( dtrParams, inRecs, settingsFile ):
    outLines = []
    for key, value in dtrParams.items():
         print( key, '=', value, file=settingsFile )

    for inRec in inRecs:
        details = inRec
        #iid = details['instanceId']
        #logger.info( 'NCSC Inst details %s', details )
        if 'commandState' in details and details['commandState'] != 'good':
            continue
        if details['state'] == 'started':
            if 'ssh' in details:
                host = details['ssh']['host']
                port = details['ssh']['port']
                user = details['ssh']['user']
                outLine = "node = %s@%s:%s" % (
                        user, host, port
                )
                #print( outLine)
                outLines.append( outLine )
                #print( "node = root@%s:%s" % (
                #        host, port
                #    ))
    for outLine in sorted( outLines):
        print( outLine, file=settingsFile )

def magickConvert( srcFilePath, destFilePath ):
    colorSpace = 'sRGB'  # fancier version could maybe override this
    # using magick convert (rather than just 'convert') means we are expecting image v 7.x
    # could use -depth 8 to produce smaller files (default is 16)
    # would use 'magick convert' instead of just 'convert' to force use of version >= 7.0
    cmd = [
        'convert', srcFilePath,
        '-colorspace', colorSpace,
        destFilePath
    ]
    #logger.debug( 'conversion cmd %s', ' '.join(cmd)  )
    try:
        subprocess.check_call( cmd,
            stdout=sys.stderr, stderr=subprocess.STDOUT
            )
    except Exception as exc:
        logger.warning( 'magick convert call threw exception (%s) %s',type(exc), exc )

def output_reader(proc):
    for line in iter(proc.stdout.readline, b''):
        print('<dtr>: {0}'.format(line.decode('utf-8')), end='', file=sys.stderr)

def startDtr( dtrDirPath, workingDir, flush=True ):
    '''starts dtr; returns a dict with "proc" and "thread" elements '''
    dtrArg = '--flush' if flush else '--clean'
    logger.info( 'calling dtr.py' )
    result = {}
    cmd = [
        'python3', '-u', dtrDirPath+'/dtr.py', dtrArg
    ]
    try:
        logger.info( 'cmd %s', cmd )
        proc = subprocess.Popen( cmd, cwd=workingDir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
        #proc = subprocess.Popen( cmd, stderr=subprocess.PIPE, stdout=subprocess.STDOUT )
        logger.info( 'started dtr.py' )
        result['proc'] = proc
        t = threading.Thread(target=output_reader, args=(proc,))
        result['thread'] = t
        t.start()
    except subprocess.CalledProcessError as exc:
        # this section never runs because Popen does not raise this exception
        logger.error( 'return code: %d %s', exc.returncode,  exc.output )
        raise  # TODO raise a more helpful specific type of error
    finally:
        return result

def triage( statuses ):
    ''' separates good tellInstances statuses from bad ones'''
    goodOnes = []
    badOnes = []

    for status in statuses:
        if isinstance( status['status'], int) and status['status'] == 0:
            goodOnes.append( status['instanceId'])
        else:
            badOnes.append( status )
    return (goodOnes, badOnes)

if __name__ == "__main__":
    # configure logger formatting
    logFmt = '%(asctime)s %(levelname)s %(module)s %(funcName)s %(message)s'
    logDateFmt = '%Y/%m/%d %H:%M:%S'
    formatter = logging.Formatter(fmt=logFmt, datefmt=logDateFmt )
    logging.basicConfig(format=logFmt, datefmt=logDateFmt)
    ncs.logger.setLevel(logging.INFO)
    logger.setLevel(logging.INFO)
    tellInstances.logger.setLevel(logging.INFO)
    logger.debug('the logger is configured')

    ap = argparse.ArgumentParser( description=__doc__,
        fromfile_prefix_chars='@', formatter_class=argparse.ArgumentDefaultsHelpFormatter )
    ap.add_argument( 'blendFilePath', help='the .blend file to render' )
    ap.add_argument( '--authToken', required=True, help='the NCS authorization token to use' )
    ap.add_argument( '--filter', help='json to filter instances for launch' )
    ap.add_argument( '--instTimeLimit', type=int, default=900, help='amount of time (in seconds) installer is allowed to take on instances' )
    ap.add_argument( '--jobId', help='to identify this job' )
    ap.add_argument( '--launch', type=boolArg, default=True, help='to launch and terminate instances' )
    ap.add_argument( '--nWorkers', type=int, default=1, help='the # of worker instances to launch (or zero for all available)' )
    ap.add_argument( '--sshAgent', type=boolArg, default=False, help='whether or not to use ssh agent' )
    ap.add_argument( '--sshClientKeyName', help='the name of the uploaded ssh client key to use (default is random)' )
    ap.add_argument( '--timeLimit', type=int, help='time limit (in seconds) for the whole job',
        default=24*60*60 )
    ap.add_argument( '--useCompositor', type=boolArg, default=True, help='whether or not to use blender compositor' )
    # dtr-specific args
    ap.add_argument( '--width', type=int, help='the width (in pixels) of the output',
        default=960 )
    ap.add_argument( '--height', type=int, help='the height (in pixels) of the output',
        default=540 )
    ap.add_argument( '--blocks_user', type=int, help='the number of blocks to partition the image (or zero for "auto"',
        default=0 )
    ap.add_argument( '--fileType', choices=['PNG', 'OPEN_EXR'], help='the type of output file',
        default='PNG' )
    ap.add_argument( '--frame', type=int, help='the frame number to render',
        default=1 )
    ap.add_argument( '--seed', type=int, help='the blender cycles noise seed',
        default=0 )
    args = ap.parse_args()
    #logger.debug('args: %s', args)

    signal.signal( signal.SIGTERM, sigtermHandler )
    myPid = os.getpid()
    logger.info('procID: %s', myPid)

    #logger.info( '--filter arg <%s>', args.filter )

    dataDirPath = os.path.abspath('./data' )
    launchedJsonFilePath = dataDirPath+'/launched.json'
    dtrSettingsFilePath = dataDirPath + '/user_settings.conf'
    settingsJsonFilePath = dataDirPath + '/settings.json'
    dtrDirPath = os.path.expanduser('~/dtr')

    launchWanted = args.launch

    startTime = time.time()
    eventTimings = []
    #starterTiming = eventTiming.eventTiming('startup')
    #starterTiming.finish()
    #eventTimings.append(starterTiming)

    os.makedirs( dataDirPath, exist_ok=True )

    nWorkersWanted = args.nWorkers
    if launchWanted:
        # overwrite the launchedJson file as empty list, so we won't have problems with stale contents
        with open( launchedJsonFilePath, 'w' ) as outFile:
            json.dump( [], outFile )
    resultsLogFilePath = os.path.join(dataDirPath, os.path.basename( __file__ ) + '.jlog' )
    # truncate the resultsLogFile
    with open( resultsLogFilePath, 'wb' ) as xFile:
        pass # xFile.truncate()

    dtrStatus = None
    try:
        masterSpecs = None
        if launchWanted:
            logger.info( 'launching using filters: %s', args.filter )
            nAvail = ncs.getAvailableDeviceCount( args.authToken, filtersJson=args.filter )
            if nWorkersWanted > (nAvail + 5):
                logger.error( 'not enough devices available (%d requested)', nWorkersWanted )
                sys.exit(1)
            if nWorkersWanted == 0:
                logger.info( '%d devices available to launch', nAvail )
                nWorkersWanted = nAvail
            if args.sshClientKeyName:
                sshClientKeyName = args.sshClientKeyName
            else:
                keyContents = loadSshPubKey()
                randomPart = str( uuid.uuid4() )[0:13]
                keyContents += ' #' + randomPart
                sshClientKeyName = 'bfr_%s' % (randomPart)
                respCode = ncs.uploadSshClientKey( args.authToken, sshClientKeyName, keyContents )
                if respCode < 200 or respCode >= 300:
                    logger.warning( 'ncs.uploadSshClientKey returned %s', respCode )
                    sys.exit( 'could not upload SSH client key')

            #TODO handle error from launchInstances
            rc = launchInstances( args.authToken, nWorkersWanted,
                sshClientKeyName, launchedJsonFilePath, filtersJson=args.filter )
            if rc:
                logger.debug( 'launchInstances returned %d', rc )
            # delete sshClientKey only if we just uploaded it
            if sshClientKeyName != args.sshClientKeyName:
                logger.info( 'deleting sshClientKey %s', sshClientKeyName)
                ncs.deleteSshClientKey( args.authToken, sshClientKeyName )

        loadedInstances = None
        with open( launchedJsonFilePath, 'r' ) as jsonFile:
            loadedInstances = json.load(jsonFile)  # a list of dicts
        startedInstances = [inst for inst in loadedInstances if inst['state'] == 'started' ]
        goodInstances = startedInstances

        if launchWanted:  #launchWanted:
            with open( os.path.expanduser('~/.ssh/known_hosts'), 'a' ) as khFile:
                jsonToKnownHosts.jsonToKnownHosts( goodInstances, khFile )

        wellInstalled = []
        if not sigtermSignaled():
            installerCmd = 'sudo apt-get -qq update && sudo apt-get -qq -y install blender > /dev/null'
            # tell them to ping
            stepTiming = eventTiming.eventTiming('tellInstances install')
            logger.info( 'calling tellInstances to install on %d instances', len(startedInstances))
            stepStatuses = tellInstances.tellInstances( startedInstances, installerCmd,
                resultsLogFilePath=resultsLogFilePath,
                download=None, downloadDestDir=None, jsonOut=None, sshAgent=args.sshAgent,
                timeLimit=min(args.instTimeLimit, args.timeLimit), upload=None,
                knownHostsOnly=True
                )
            # restore our handler because tellInstances may have overridden it
            signal.signal( signal.SIGTERM, sigtermHandler )
            if not stepStatuses:
                logger.warning( 'no statuses returned from installer')
            (goodOnes, badOnes) = triage( stepStatuses )
            stepTiming.finish()
            eventTimings.append(stepTiming)
            logger.info( 'stepStatuses %s', stepStatuses )
        if goodOnes and not sigtermSignaled():
            goodInstances = [inst for inst in goodInstances if inst['instanceId'] in goodOnes ]

            blocks_user = args.blocks_user
            if not blocks_user:
                if nWorkersWanted == 1:
                    blocks_user = 1
                else:
                    # nBlocks computation is arbitrary, based on limited experience
                    diag = math.sqrt( args.width * args.height )
                    blocks_user = diag / 12
                    #blocks_user = args.height / 9  # arbitrary, based on limited experience
                    blocks_user = int( max( blocks_user, len(goodInstances)*1 ) )

            dtrParams = {
                'image_x': args.width,
                'image_y': args.height,
                'blocks_user': blocks_user,
                'filetype': args.fileType,
                'frame': args.frame,
                'seed': args.seed,
            }
            if args.useCompositor:
                dtrParams['filetype'] = 'OPEN_EXR'
            extensions = {'PNG': 'png', 'OPEN_EXR': 'exr'}
            prerenderedFileName = 'composite_seed_%d.%s' % \
                (args.seed, extensions[dtrParams['filetype']])
            outFilePattern = 'rendered_frame_######_seed_%d.%s'%(args.seed,extensions[args.fileType])
            outFileName = outFilePattern.replace( '######', '%06d' % args.frame )

            with open( dtrSettingsFilePath, 'w' ) as settingsFile:
                generateDtrConf( dtrParams, goodInstances, settingsFile )

            settingsToSave = dtrParams.copy()
            settingsToSave['outFileName'] = outFileName
            with open( settingsJsonFilePath, 'w' ) as settingsFile:
                json.dump( settingsToSave, settingsFile )

            # copy some files into a working dir, because dtr has no args for them
            shutil.copyfile( dtrDirPath+'/bench.blend', dataDirPath+'/bench.blend' )
            if os.path.abspath( args.blendFilePath ) \
                != os.path.abspath( dataDirPath+'/render.blend' ):
                shutil.copyfile( args.blendFilePath, dataDirPath+'/render.blend' )

            masterSpecs = startDtr( dtrDirPath, workingDir=dataDirPath, flush=True )
            if masterSpecs:
                proc = masterSpecs['proc']
                deadline = startTime + args.timeLimit
                while time.time() < deadline:
                    proc.poll() # sets proc.returncode
                    if proc.returncode != None:
                        dtrStatus = proc.returncode
                        if proc.returncode != 0:
                            logger.error( 'dtr gave an unexpected returnCode %d', proc.returncode )
                        break
                    if sigtermSignaled():
                        logger.info( 'signaling dtr')
                        proc.send_signal( signal.SIGTERM )
                        try:
                            logger.info( 'waiting dtr')
                            proc.wait(timeout=300)
                            if proc.returncode:
                                logger.warning( 'dtr return code %d', proc.returncode )
                        except subprocess.TimeoutExpired:
                            logger.warning( 'dtr did not terminate in time' )
                        break
                    time.sleep(.5)
                if proc.returncode == None:
                    logger.info('at end of polling loop, no return code' )
                    dtrStatus = 124  # linux convention for timeout
                    proc.send_signal( signal.SIGTERM )
                    try:
                        logger.info( 'waiting dtr')
                        proc.wait(timeout=300)
                        if proc.returncode:
                            logger.warning( 'dtr return code %d', proc.returncode )
                    except subprocess.TimeoutExpired:
                        logger.warning( 'dtr did not terminate in time' )

    except KeyboardInterrupt:
        logger.warning( '(ctrl-c) received, will shutdown gracefully' )
    except Exception as exc:
        logger.error( 'an exception occurred; will try to shutdown gracefully', exc_info=True )
    if launchWanted:
        # get instances from json file, to see which ones to terminate
        launchedInstances = []
        with open( launchedJsonFilePath, 'r') as jsonInFile:
            try:
                launchedInstances = json.load(jsonInFile)  # an array
            except Exception as exc:
                logger.warning( 'could not load json (%s) %s', type(exc), exc )
        if len( launchedInstances ):
            launcherJob = launchedInstances[0].get('job')
            if launcherJob:
                logger.info( 'calling terminateJobInstances for job "%s"', launcherJob )
                ncs.terminateJobInstances( args.authToken, launcherJob )
            else:
                terminateThese( args.authToken, launchedInstances )
            # purgeKnownHosts works well only when known_hosts is not hashed
            cmd='purgeKnownHosts.py %s > /dev/null' % (launchedJsonFilePath)
            try:
                subprocess.check_call( cmd, shell=True )
            except Exception as exc:
                logger.error( 'purgeKnownHosts threw exception (%s) %s',type(exc), exc )

    # run blender compositor if dtr succeeded
    if dtrStatus == 0:
        retCode = None
        if args.useCompositor:
            cmd = [
                'blender', '-b', '-noaudio', dataDirPath+'/render.blend',
                '-P', scriptDirPath()+'/composite_bpy.py',
                '-o',  dataDirPath+'/'+outFilePattern,
                '-f', str(args.frame), '--', '--prerendered', prerenderedFileName
            ]
            logger.info( 'compositing cmd %s', cmd )
            try:
                retCode = subprocess.call( cmd,
                    stdout=sys.stderr, stderr=subprocess.STDOUT
                    )
            except Exception as exc:
                logger.warning( 'blender composite_bpy call threw exception (%s) %s',type(exc), exc )
            # retCode 90 indicates that there was no compositor graph
            if retCode == 90:
                magickConvert( dataDirPath+'/'+prerenderedFileName, dataDirPath+'/'+outFileName )
            #TODO: do something about other non-zero return codes
        if (args.useCompositor==False):
            # rename dtr/imagemagick output to the more desirable fileName
            os.rename( dataDirPath+'/'+prerenderedFileName, dataDirPath+'/'+outFileName )
    # clean up .blend files that were copied
    if os.path.isfile( dataDirPath+'/render.blend' ):
        try:
            # delete render.blend, except in certain conditions; this may be seen as risky
            if dataDirPath != dtrDirPath:
                os.remove( dataDirPath+'/render.blend' )
        except Exception as exc:
            logger.warning( 'exception while deleting render.blend (%s) %s', type(exc), exc, exc_info=False )
    if os.path.isfile( dataDirPath+'/bench.blend' ):
        try:
            if dataDirPath != dtrDirPath:
                os.remove( dataDirPath+'/bench.blend' )
        except Exception as exc:
            logger.warning( 'exception while deleting bench.blend (%s) %s', type(exc), exc, exc_info=False )

    if dtrStatus != None:
        rc = dtrStatus
    elif sigtermSignaled():
        rc = 128 + 15  # linux convention when exiting due to signal 15 (sigterm)
    else:
        rc=1
    logger.info( 'finished with rc %d', rc)
    sys.exit(rc)


from django.urls import path
from . import views

app_name = 'cinema'
urlpatterns = [
    path('', views.movies, name='movies'),
    path('persons/', views.persons, name='persons'),
    path('person_detail/<str:imdb_id>/', views.person_detail, name='person_detail'),
    path('<str:imdb_id>/', views.movie_detail, name='movie_detail'),
    path('<str:imdb_id>/edit_save=<int:edit_save>/', views.movie_detail, name='movie_detail'),
    path('<str:imdb_id>/edit_save=<int:edit_save>/b_page_number=<int:b_page_number>', views.movie_detail, name='movie_detail'),
]


from flask import Flask, jsonify, Request
from flask_cors import CORS
import os
import current_patient_service as cps
import json

app = Flask(__name__)
CORS(app)

@app.route("/api/get_shap_interpretation_by_session_id/<session_id>",
            methods=["GET"])
def get_cardiac_cpet_intepretation_by_id(session_id):
    """API that returns a base64 of the force plot and custom shap summary

    This method validates the input and calls a method that will process a
    list of values that represents the time and voltage of an ecg signal.
    It plots the information and enconde it as a base64 string and without
    storing it into the database.

    Args:
        in_data (str): session id

    Returns
        str,str, int: SHAP and force plot image as a base64 str, 200
        str, int: Error Message, 400
    """
    try:
        shap_custom_result = cps.get_interpretation_images_by_id(session_id)
        #res, code = hr.get_record_by_patient_id(patient_id)
        return json.dumps(vars(shap_custom_result[0])), 200
    except Exception as e:
        print(e)
        return "Unexpected error", 400
    pass

@app.route("/api/get_dynamic_record_by_session_id/<session_id>",
            methods=["GET"])
def get_dynamic_cpet_by_session_id(session_id):
    """API that retrieves all the cpet reocrds of a session

    This method validates the input and calls a method that will retrieve
    a list of all the health records of a patient, if there is one exists.
    The result will be a list of dictionaries containing the information

    Args:
        session_id (str): The id of the health record

    Returns
        `list` of `dict`, int: All patient's medical records, 200
        str, int: Error Message, 400
    """
    try:
        result = cps.get_dynamic_cpet_record_by_session_id(session_id)
        #res, code = hr.get_record_by_patient_id(patient_id)
        return json.dumps(vars(result[0])), 200
    except Exception as e:
        print(e)
        return "Unexpected error", 400
    pass

@app.route("/api/get_record_by_patient_id/<session_id>",
           methods=["GET"])
def get_cpet_record_by_session_id(session_id):
    """API that retrieves all the cpet reocrds of a session

    This method validates the input and calls a method that will retrieve
    a list of all the health records of a patient, if there is one exists.
    The result will be a list of dictionaries containing the information

    Args:
        session_id (str): The id of the health record

    Returns
        `list` of `dict`, int: All patient's medical records, 200
        str, int: Error Message, 400
    """
    try:
        result = cps.get_cpet_record_by_session_id(session_id)
        return json.dumps(vars(result[0])), 200
    except Exception as e:
        print(e)
        return "Unexpected error", 400

@app.route('/')
def root():
    return "Hello World!"

if __name__ == "__main__":
        # This is used when running locally only. When deploying to Google App
    # Engine, a webserver process such as Gunicorn will serve the app. This
    # can be configured by adding an `entrypoint` to app.yaml.
    # Flask's development server will automatically serve static files in
    # the "static" directory. See:
    # http://flask.pocoo.org/docs/1.0/quickstart/#static-files. Once deployed,
    # App Engine itself will serve those files as configured in app.yaml.
    app.run(host='127.0.0.1', port=8080, debug=True)
    #app.run(host="0.0.0.0", port=5000)


from typing import Optional

import click
from pipelines import (
    seq_classifier_train_eval_pipeline,
    token_classifier_train_eval_pipeline,
)
from steps import (
    data_importer,
    load_tokenizer,
    sequence_classifier_tokenization,
    sequence_evaluator,
    sequence_trainer,
    token_classification_tokenization,
    token_evaluator,
    token_trainer,
)
from steps.configuration import HuggingfaceParameters

@click.command()
@click.option(
    "--nlp_task",
    type=click.Choice(["token-classification", "sequence-classification"]),
    default="sequence-classification",
    help="Name NLP task i.e. token-classification, sequence-classification",
)
@click.option(
    "--pretrained_model",
    default="distilbert-base-uncased",
    help="Pretrained model name from huggingface hub",
)
@click.option(
    "--batch_size",
    default=8,
    help="Batch Size for training",
)
@click.option(
    "--full_set",
    is_flag=True,
    help="By default only a very small subset of the datasets is used in order "
    "to have a quick end-to-end run. By running with the full datasets "
    "the runtime increases significantly.",
)
@click.option(
    "--epochs",
    default=1,
    help="Number of epochs for training",
)
@click.option(
    "--max_seq_length",
    default=128,
    help="Max sequence length for training",
)
@click.option(
    "--init_lr",
    default=2e-5,
    help="Initial learning rate for training",
)
@click.option(
    "--weight_decay_rate",
    default=0.01,
    help="Weight decay rate for training",
)
@click.option(
    "--text_column",
    help="Column name for text in the dataset. i.e. For sequence "
    "classification, this will be text and for token classification, "
    "this will be tokens",
)
@click.option(
    "--label_column",
    help="Column name for label in the dataset. i.e For sequence"
    " classification, this will be label and for token classification, "
    "this will be ner_tags",
)
@click.option(
    "--dataset_name",
    help="Name of the dataset to be used. i.e For sequence classification, "
    "this will be imdb and for token classification, this will be "
    "conll2003",
)
def main(
    nlp_task: str,
    pretrained_model: str,
    batch_size: int,
    epochs: int,
    full_set: bool,
    max_seq_length: int,
    init_lr: float,
    weight_decay_rate: float,
    text_column: Optional[str] = None,
    label_column: Optional[str] = None,
    dataset_name: Optional[str] = None,
):
    if nlp_task == "token-classification":
        if not text_column:
            text_column = "tokens"
        if not label_column:
            label_column = "ner_tags"
        if not dataset_name:
            dataset_name = "conll2003"

        token_classification_config = HuggingfaceParameters(
            label_all_tokens=True,
            pretrained_model=pretrained_model,
            epochs=epochs,
            batch_size=batch_size,
            max_seq_length=max_seq_length,
            init_lr=init_lr,
            weight_decay_rate=weight_decay_rate,
            dummy_run=not full_set,
            dataset_name=dataset_name,
            text_column=text_column,
            label_column=label_column,
        )
        pipeline = token_classifier_train_eval_pipeline(
            importer=data_importer(token_classification_config),
            load_tokenizer=load_tokenizer(token_classification_config),
            tokenization=token_classification_tokenization(
                token_classification_config
            ),
            trainer=token_trainer(token_classification_config),
            evaluator=token_evaluator(token_classification_config),
        )
        pipeline.run()

    elif nlp_task == "sequence-classification":
        if not text_column:
            text_column = "text"
        if not label_column:
            label_column = "label"
        if not dataset_name:
            dataset_name = "imdb"

        sequence_classification_config = HuggingfaceParameters(
            pretrained_model=pretrained_model,
            epochs=epochs,
            batch_size=batch_size,
            dummy_run=not full_set,
            max_seq_length=max_seq_length,
            init_lr=init_lr,
            weight_decay_rate=weight_decay_rate,
            dataset_name=dataset_name,
            text_column=text_column,
            label_column=label_column,
        )
        pipeline = seq_classifier_train_eval_pipeline(
            importer=data_importer(sequence_classification_config),
            load_tokenizer=load_tokenizer(sequence_classification_config),
            tokenization=sequence_classifier_tokenization(
                sequence_classification_config
            ),
            trainer=sequence_trainer(sequence_classification_config),
            evaluator=sequence_evaluator(sequence_classification_config),
        )
        pipeline.run()

if __name__ == "__main__":
    main()


import json
from datetime import datetime

import scrapy

from util import get_dates_between
from bs4 import BeautifulSoup
import csv

class BaseballSpider(scrapy.Spider):
    name = "baseball"
    start_urls = ["http://sports.news.naver.com/"]

    def __init__(self):
        start_datetime = datetime(2022, 1, 1)
        end_datetime = datetime(2022, 12, 31)
        self.target_dates = get_dates_between(start_datetime, end_datetime)
        self.article_list_url = "https://sports.news.naver.com/kbaseball/news/list?isphoto=N&date={date}&page={page}"
        self.article_url = "https://sports.news.naver.com/news?oid={oid}&aid={aid}"
        self.output_file = open("./baseball.csv", "w")
        self.writer = csv.writer(self.output_file)
        self.writer.writerow(["url", "datetime", "title", "content"])

    def parse(self, response):
        for target_date in self.target_dates:
            target_url = self.article_list_url.format(date=target_date, page=1)
            req = scrapy.Request(
                url=target_url,
                callback=self.parse_total_pages
            )
            req.meta["date"] = target_date
            yield req

    def parse_total_pages(self, response):
        date = response.meta["date"]
        resp_json = json.loads(response.text)
        total_pages = resp_json["totalPages"]
        for i in range(total_pages):
            req = scrapy.Request(
                url=self.article_list_url.format(date=date, page=i+1),
                callback=self.parse_article_list
            )
            req.meta["date"] = date
            yield req

    def parse_article_list(self, response):
        date = response.meta["date"]
        resp_json = json.loads(response.text)
        for item in resp_json["list"]:
            oid = item["oid"]
            aid = item["aid"]
            req = scrapy.Request(
                url=self.article_url.format(oid=oid, aid=aid),
                callback=self.parse_article
            )
            req.meta["date"] = date
            yield req

    def parse_article(self, response):
        url = response.request.url
        date = response.meta["date"]

        def _remove_tags(parent_soup, target_tag):
            tags = parent_soup.find_all(target_tag)
            for tag in tags:
                tag.decompose()

        soup = BeautifulSoup(response.text, "lxml")
        title = soup.find("h4", class_="title").get_text()
        content_soup = soup.find("div", id="newsEndContents")
        _remove_tags(content_soup, "p")
        _remove_tags(content_soup, "div")
        _remove_tags(content_soup, "em")
        _remove_tags(content_soup, "span")
        content = content_soup.get_text().strip()
        self.writer.writerow([url, date, title, content])


import os
import argparse

parser = argparse.ArgumentParser()
parser.add_argument('--checkpoint')
parser.add_argument('--device', default = 'cuda', help = 'TODO: proper device choosing')
parser.add_argument('--iterations', type = int, default = 16)
parser.add_argument('--iterations-warmup', type = int, default = 16)
parser.add_argument('--frontend', action = 'store_true')
parser.add_argument('--fp16', choices = ['', 'O0', 'O1', 'O2', 'O3'], default = None)
parser.add_argument(
	'--num-input-features', default = 64, help = 'num of mel-scale features produced by logfbank frontend'
)
parser.add_argument('--sample-rate', type = int, default = 8_000, help = 'for frontend')
parser.add_argument('--window-size', type = float, default = 0.02, help = 'for frontend, in seconds')
parser.add_argument('--window-stride', type = float, default = 0.01, help = 'for frontend, in seconds')
parser.add_argument('--lang', default = 'ru')
parser.add_argument(
	'--window', default = 'hann_window', choices = ['hann_window', 'hamming_window'], help = 'for frontend'
)
parser.add_argument('--model', default = 'JasperNetBig')
parser.add_argument('--onnx')
parser.add_argument('--onnx-output-node-name', type = str, default = 'logits')
parser.add_argument('--stft-mode', choices = ['conv', ''], default = '')
parser.add_argument('-B', type = int, default = 256)
parser.add_argument('-T', type = float, default = 5.12)
parser.add_argument('--profile-cuda', action = 'store_true')
parser.add_argument('--save-cudnn-cublas-logs', action = 'store_true', help='Save CUDNN CUBLAS logs')
parser.add_argument('--cudnn-logdest', type=str, default = 'cudnn_log.txt', help='Destination path to CUDNN logs')
parser.add_argument('--cublas-logdest', type=str, default = 'cublas_log.txt', help='Destination path to CUBLAS logs')

parser.add_argument('--profile-pyprof', action = 'store_true')
parser.add_argument('--profile-autograd')
parser.add_argument('--data-parallel', action = 'store_true')
parser.add_argument('--backward', action = 'store_true')
parser.add_argument('--input-time-dim-multiple', type = int, default = 128)

parser.add_argument('--output-path', '-o')

args = parser.parse_args()

if args.save_cudnn_cublas_logs:
	os.environ["CUDNN_LOGINFO_DBG"] = '1'
	os.environ["CUBLAS_LOGINFO_DBG"] = '1'
	os.environ["CUDNN_LOGDEST_DBG"] = args.cudnn_logdest
	os.environ["CUBLAS_LOGDEST_DBG"] = args.cublas_logdest

import gc
import math
import time
import torch
import torch.cuda.profiler
import onnxruntime
import models
import datasets
import utils

checkpoint = torch.load(args.checkpoint, map_location = 'cpu') if args.checkpoint else None
if checkpoint:
	args.model, args.lang, args.sample_rate, args.window_size, args.window_stride, args.window, args.num_input_features = map(
		checkpoint['args'].get,
		['model', 'lang', 'sample_rate', 'window_size', 'window_stride', 'window', 'num_input_features'])

use_cuda = 'cuda' in args.device

if args.onnx:
	onnxruntime_session = onnxruntime.InferenceSession(args.onnx)

	def load_batch_to_device(batch, device):
		batch = batch.numpy()
		io_binding = onnxruntime_session.io_binding()
		batch_ortvalue = onnxruntime.OrtValue.ortvalue_from_numpy(batch, device, 0)
		io_binding.bind_input(
			name = 'x',
			device_type = batch_ortvalue.device_name(),
			device_id = 0,
			element_type = batch.dtype,
			shape = batch_ortvalue.shape(),
			buffer_ptr = batch_ortvalue.data_ptr()
		)
		io_binding.bind_output(args.onnx_output_node_name, device)
		return io_binding

	load_batch = lambda batch: load_batch_to_device(batch, args.device)
	model = lambda io_binding: onnxruntime_session.run_with_iobinding(io_binding)

else:
	frontend = models.LogFilterBankFrontend(
		args.num_input_features,
		args.sample_rate,
		args.window_size,
		args.window_stride,
		args.window,
		stft_mode = args.stft_mode,
	) if args.frontend else None
	model = getattr(models, args.model)(
		args.num_input_features, [len(labels)],
		frontend = frontend,
		dict = lambda logits,
		log_probs,
		olen,
		**kwargs: logits[0]
	)
	if checkpoint:
		model.load_state_dict(checkpoint['model_state_dict'], strict = False)
	model.to(args.device)

	if not args.backward:
		model.eval()
		model.fuse_conv_bn_eval()

	model, *_ = models.data_parallel_and_autocast(model, opt_level = args.fp16, data_parallel = args.data_parallel)
	load_batch = lambda x: x.to(args.device, non_blocking = True)

tictoc = lambda: (use_cuda and torch.cuda.synchronize()) or time.time()

batch_shape = [args.B, int(args.T * args.sample_rate)
				] if args.frontend else [args.B, args.num_input_features, int(args.T / args.window_stride)]

if args.input_time_dim_multiple and batch_shape[-1] % args.input_time_dim_multiple != 0:
	batch_shape[-1] = int(math.ceil(batch_shape[-1] / args.input_time_dim_multiple) * args.input_time_dim_multiple)

example_time = batch_shape[-1] / args.sample_rate if args.frontend else batch_shape[-1] * args.window_stride

batch = torch.rand(*batch_shape)

print(args)
print()
print(
	'batch {} | stride {:.02f} sec | audio {:.02f} sec'.format(
		'x'.join(map(str, batch_shape)), args.window_stride, args.B * example_time
	)
)
print()

print('Warming up for', args.iterations_warmup, 'iterations')
tic_wall = tictoc()
if use_cuda:
	torch.backends.cudnn.benchmark = True

torch.set_grad_enabled(args.backward)
loaded_batch = load_batch(batch)
for i in range(args.iterations_warmup):
	y = model(loaded_batch)
	if args.backward:
		y.sum().backward()

print('Warmup done in {:.02f} wall clock seconds'.format(tictoc() - tic_wall))
print()

if args.profile_pyprof or args.profile_cuda:
	import pyprof
	pyprof.nvtx.nvmarker.patch_apex_module = lambda modstr, old = pyprof.nvtx.nvmarker.patch_apex_module: old(
			modstr) if modstr != 'apex.contrib.multihead_attn' else None  # coused by this bug https://github.com/NVIDIA/apex/issues/958 https://github.com/NVIDIA/PyProf/issues/167
	pyprof.nvtx.init()
if args.profile_cuda:
	torch.autograd.profiler.emit_nvtx()
	torch.cuda.profiler.start()
if args.profile_autograd:
	autograd_profiler = torch.autograd.profiler.profile(
		use_cuda = use_cuda, enabled = True, profile_memory = True, record_shapes = True
	)
	autograd_profiler.__enter__()

print('Starting benchmark for', args.iterations, 'iterations:', 'fwd', '+ bwd' if args.backward else '')
tic_wall = tictoc()
times_fwd, times_bwd, fragmentation, global_device_memory_utilisation = \
	torch.zeros(args.iterations), torch.zeros(args.iterations), \
		torch.zeros(args.iterations), torch.zeros(args.iterations)

loaded_batch = load_batch(batch)
for i in range(args.iterations):
	tic = tictoc()
	y = model(loaded_batch)
	toc = tictoc()
	global_device_memory_utilisation[i] = utils.compute_global_device_memory_utilisation()
	fragmentation[i] = utils.compute_memory_fragmentation()
	if args.backward:
		y.sum().backward()
	tac = tictoc()
	times_fwd[i] = toc - tic
	times_bwd[i] = tac - toc
	y = None

mem_reserved = torch.cuda.max_memory_reserved(args.device) if use_cuda else 0
mem_allocated = torch.cuda.max_memory_allocated(args.device) if use_cuda else 0
print('Benchmark done in {:.02f} wall clock seconds'.format(tictoc() - tic_wall))
print()

if args.profile_autograd:
	autograd_profiler.__exit__(None, None, None)
	autograd_profiler.export_chrome_trace(args.profile_autograd)
print(f'B: {args.B}, T: {args.T}')
print(
	'load+fwd {:.02f} msec | bwd {:.02f} msec | cudamemreserved {:.02f} mb | cudamemallocated {:.02f} mb | '
	'cudamemutilization: {:.02f} | global_dev_memutilization: {:.02f} | rtf: {:.02f}'
	.format(
		float(times_fwd.mean()) * 1e3,
		float(times_bwd.mean()) * 1e3,
		mem_reserved * 1e-6,
		mem_allocated * 1e-6,
		float(fragmentation.mean()),
		float(global_device_memory_utilisation.mean()),
		float(args.B * example_time * args.iterations / times_fwd.sum())
	)
)

if args.output:
	with open(args.output, 'a+') as f:
		message = f'{args.B}\t' \
		           f'{args.T}\t' \
		           f'{times_fwd.mean()}\t' \
		           f'{times_fwd.sum()}\t' \
		           f'{fragmentation.mean()}\t' \
		           f'{args.B * example_time * args.iterations / 3600.0}\t' \
		           f'{args.B * example_time * args.iterations / times_fwd.sum()}\t' \
		           f'{args.checkpoint}\t' \
		           f'{args.onnx}\t' \
		           f'\n'
		f.write(message)


#!/usr/bin/env python2

import numpy.random as npr
from collections import defaultdict
from collections import deque
from random import sample
from math import ceil
import math_tools
import json

##########################################
#
# COMMUNITIES
#
# This class contains the community information of the network.
# This information can eithe be read from the LFR-output or
# deduced from the affinity output of the c++ algorithm.
#
# Community information can be accessed in two different ways:
# * The dict "vertexToCommunities" maps each vertex to the communities it belongs to.
# * The dict "communityToVertices" maps each community to the vertices which are part of it.

class Communities(object):

    def __init__(self):
        self.vertexToCommunities = None
        self.communityToVertices = None
        self.numberOfCommunities = None

    def reverseMapping(self, inputDict):
        outputDict = defaultdict(list)
        # remap value list as keys, keys as values
        for key, listOfKey in inputDict.iteritems():
            for element in listOfKey:
                outputDict[element].append(key)
        return outputDict

    # input and output
    # ----------------

    # Read community information from the LFR community file.
    # The input is a file which where each line consists
    # of a nodeid and then a list of communities this node belongs to
    # Ids of nodes and communities in the LFR file start at 1,
    # we subtract 1 to let those ids start at 0.
    def readCommunitiesLFR(self, filename):
        self.vertexToCommunities = defaultdict(list)
        with open (filename, "r") as f:
            belongings = [[(int(x) - 1) for x in line.split()] for line in f.readlines()]
            for belonging in belongings:
                self.vertexToCommunities[belonging[0]] = belonging[1:]
        self.communityToVertices = self.reverseMapping(self.vertexToCommunities)
        self.numberOfCommunities = len(self.communityToVertices)

    #the same as readCommunitiesLFR but reads our file format instead
    #TODO: what format are you talking about?
    #TODO: this function seems buggy: vertices are not cast to int
    def readCommunitiesOurFormat(self, filename):
        self.communityToVertices = defaultdict(list)
        with open (filename, "r") as f:
            for community,line in enumerate(f):
                for vertex in line.split():
                    self.communityToVertices[community].append(vertex)
        self.vertexToCommunities = self.reverseMapping(self.communityToVertices)
        self.numberOfCommunities = len(self.communityToVertices)

    # Write output community file.
    # Each line represents one community and lists all the vertices in this community.
    def writeCommunitesOurFormat(self, filename):
        with open (filename, "w") as f:
            for vertices in self.communityToVertices.values():
                f.write(" ".join([str(x) for x in vertices]) + "\n")

    # Write seed-information to file in affinity-fileformat.
    # seeds is a set of nodeids
    def writeSeedsOurFormat(self, seeds, filename):
        if seeds is None:
            raise Exception("seeds need to be generated first")
        with open (filename, "w") as f:

            f.write("{0} {1}".format(len(seeds), self.numberOfCommunities))
            for seed in seeds:
                tmp = [0] * self.numberOfCommunities
                for community in self.vertexToCommunities[seed]:
                    tmp[community] = 1
                f.write("\n" + str(seed) + " " + " ".join([str(x) for x in tmp]))

    # methods for generating seeds
    # ----------------------------

    # Generate seed nodes from a community object.
    # Pick a fraction of "seedFraction" nodes from each community.
    # The number of seeds per community is at least 1 and rounded to the next bigger integer
    def generateSeedsUniformly(self, seedFraction):
        seeds = set()
        for c in self.communityToVertices:
            if seedFraction == 0:
                seedCount = 1
            else:
                seedCount = int(ceil(seedFraction * len(self.communityToVertices[c])))
            seeds = seeds.union(sample(self.communityToVertices[c], seedCount))
        return seeds

    def generateSeedsDegreeBased(self, graph, seedFraction):
        seeds = set()

        for c, vertices in self.communityToVertices.iteritems():
            communityDegree = 0

            probabilities = list()
            for vertex in vertices:
                communityDegree += len(graph.vertexToNeighbours[vertex])

            for vertex in vertices:
                probabilities.append(len(graph.vertexToNeighbours[vertex]) / float(communityDegree))

            if seedFraction == 0:
                seedCount = 1
            else:
                seedCount = int(ceil(seedFraction * len(self.communityToVertices[c])))

            tmp = math_tools.choice(vertices, seedCount, replace=False, p=probabilities)
            seeds = seeds.union(tmp)
        return seeds

    # stuff
    # ----------------------------

    #Find position of biggest gap
    def _getGap(self,affinityVector):
        affinitytuple = list()

        #create new data structure with community id and correspodning affinity
        for i, affinity in enumerate(affinityVector):
            affinitytuple.append((i,affinity))

        affinitytuple = sorted(affinitytuple,key=lambda item: item[1], reverse=True)

        #Find position of maximum gap
        maxDiff = 0
        maxDiffPosition = -1
        for i in xrange(0,len(affinitytuple[:-1])):
            currentDiff = affinitytuple[i][1] - affinitytuple[i+1][1]
            if currentDiff > maxDiff:
                maxDiff = currentDiff
                maxDiffPosition = i

        first_half = affinitytuple[:maxDiffPosition+1]
        communities = list()
        for element in first_half:
            communities.append(element[0])
        return communities, maxDiffPosition, maxDiff

    def _getJSONOfVertex(self,affinities, groundTruth, vertex):
        affinity_vector = affinities.vertexToAffinities[vertex]
        deviation = math_tools.standartDeviation(affinity_vector)
        communities_according_to_gap,gap_position ,gap_size = self._getGap(affinity_vector)

        actual_communities = groundTruth.vertexToCommunities[vertex]
        detected_communities = self.vertexToCommunities[vertex]

        return {
            "nodeid":vertex,
            "actual_communities":actual_communities,
            "actual_number_of_communities":len(actual_communities),
            "detected_communities":detected_communities,
            "detected_number_of_communities":len(detected_communities),

            "gap_position":gap_position,
            "standard_deviation":deviation,
            "affinities":affinity_vector,
            "gap_size":gap_size,
            "communities_according_to_gap":communities_according_to_gap}

    def writeJSONfile(self, filename, affinities, groundTruth):
        output = list()
        for vertex in self.vertexToCommunities:
            output.append(self._getJSONOfVertex(affinities, groundTruth, vertex))
        with open (filename, "w") as f:
            f.write(json.JSONEncoder(indent=4).encode({"body": output}))

    # classification
    # --------------

    # Classify communities from the affinity output of the c++ algorithm.
    # Simply assign the vertex the the community with the maximum affinity-value.
    def classifyCommunities(self, affinities):
        communitiesPerVertex = 1
        self.vertexToCommunities = defaultdict(list)
        for vertex, affinities in affinities.vertexToAffinities.iteritems():
            maxIndex = max([(v,i) for i,v in enumerate(affinities)])[1]
            self.vertexToCommunities[vertex] = [maxIndex]
        self.communityToVertices = self.reverseMapping(self.vertexToCommunities)
        self.numberOfCommunities = len(self.communityToVertices)

    # Classify communities from the affinity output of the c++ algorithm.
    # Assign communities according to "gap-strategy"
    def classifyCommunitiesWithGaps(self, affinities):
        self.vertexToCommunities = defaultdict(list)
        for vertex, affinities in affinities.vertexToAffinities.iteritems():
            communities,gapPosition,gapSize = self._getGap(affinities)
            self.vertexToCommunities[vertex] = communities
        self.communityToVertices = self.reverseMapping(self.vertexToCommunities)
        self.numberOfCommunities = len(self.communityToVertices)

    def classifyCommunitiesGroundTruth(self, affinities, groundTruth):
        self.vertexToCommunities = defaultdict(list)
        for vertex, affinities in affinities.vertexToAffinities.iteritems():
            actual_number_of_communities = len(groundTruth.vertexToCommunities[vertex])
            # Put affinities with their index (the corresponding community) in tuple
            # and sort them by first entry (affinity).
            # Then extract the second entry (the community) from each tuple.
            sortedTuples = sorted([(v,i) for i,v in enumerate(affinities)], reverse=True)
            communities = [i for (v,i) in sortedTuples][:actual_number_of_communities]
            self.vertexToCommunities[vertex] = communities
        self.communityToVertices = self.reverseMapping(self.vertexToCommunities)
        self.numberOfCommunities = len(self.communityToVertices)


lst = ['','A','E','I','O','U']
# t = []
# t = set(t)
# for j in lst:
#     for k in lst:
#         for q in lst:
#             for p in lst:
#                 for w in lst:
#                     t.add((j+k+q+p+w).strip())
# t = list(t)
# t.sort()

# def solution(word):
#     for i in t:
#         if i == word:
#             return t.index(i)

# #시간복잡도는 list t의 n번 순회 O(N)

from itertools import product
t2 = []
t2 = set(t2)
for i in product(lst,repeat = 5):
    s = ''.join(i)
    t2.add(s)
t2 = list(t2)
t2.sort()
t2.pop()
print(t2)

#중복순열   t==t2


from cProfile import label
from tkinter import*
from tkinter import ttk
from turtle import right
from PIL import Image,ImageTk

class ChatBot:
    def __init__(self,root):
        self.root=root
        self.root.title("Help Desk")
        self.root.geometry("1530x790+0+0")
        self.root.bind('<Return>',self.enter_func)

        main_frame=Frame(self.root,bd=4,bg='white',width=730)
        main_frame.pack()

        img_chat=Image.open(r"college_images\bot_icon.png")
        img_chat=img_chat.resize((250,250),Image.ANTIALIAS)
        self.photoimg=ImageTk.PhotoImage(img_chat)

        Title_label=Label(main_frame,bd=3,relief=RAISED,anchor='nw',width=1530,compound=LEFT,image=self.photoimg,text='   Ask Me Anything',font=('arial',30,'bold'),fg='white',bg='brown')
        Title_label.pack(side=TOP)

        self.scroll_y=ttk.Scrollbar(main_frame,orient=VERTICAL)
        self.text=Text(main_frame,width=65,height=15,bd=3,relief=RAISED,font=('arial',14),yscrollcommand=self.scroll_y.set)
        self.scroll_y.pack(side=RIGHT,fill=Y)
        self.text.pack()

        btn_frame=Frame(self.root,bd=4,bg='white',width=730)
        btn_frame.pack()

        label_l=Label(btn_frame,text="Type Here ...",font=('times new roman',14,'bold'),fg='brown')
        label_l.grid(row=0,column=0,padx=5,sticky=W)

        self.entry=StringVar()
        self.entry1=ttk.Entry(btn_frame,textvariable=self.entry,width=40,font=('times new roman',15,'bold'))
        self.entry1.grid(row=0,column=1,padx=5,sticky=W)

        self.send=Button(btn_frame,text="Send",command=self.send,font=('times new roman',16,'bold'),width=8,fg='white',bg='brown')
        self.send.grid(row=0,column=2,padx=5,sticky=W)

        self.clear=Button(btn_frame,text="Clear Fields",command=self.clear,font=('times new roman',14,'bold'),width=9,fg='brown')
        self.clear.grid(row=1,column=0,padx=5,sticky=W)

        self.msg=''
        self.label_ll=Label(btn_frame,text=self.msg,font=('times new roman',14,'bold'),fg='brown')
        self.label_ll.grid(row=1,column=1,padx=5,sticky=W)

    def enter_func(self,event):
        self.send.invoke()
        self.entry.set('')

    def clear(self):
        self.text.delete('1.0',END)
        self.entry.set('')

    def send(self):
        send='You: '+self.entry.get()
        self.text.insert(END,'\n'+send)
        self.text.yview(END)

        if(self.entry.get()==''):
            self.msg='Please enter some input'
            self.label_ll.config(text=self.msg,fg='brown')
        else:
            self.msg=''
            self.label_ll.config(text=self.msg,fg='brown')

        if(self.entry.get()=='hello' or self.entry.get()=='hi' or self.entry.get()=='hey'):
            self.text.insert(END,'\n'+'Bot: Hello I am a bot and I am here to answer your queries about the Attendance Management System')
        elif(self.entry.get()=='tell me about student details' or self.entry.get()=='student details' or self.entry.get()=='what is student details'):
            self.text.insert(END,'\n'+'Bot: Student Details is the section where you can add(new),update(existing),delete and view the student information and take photo sample too')
        elif(self.entry.get()=='train data' or self.entry.get()=='tell me about train data' or self.entry.get()=='what is train data'):
            self.text.insert(END,'\n'+'Bot: Train data is a section where you can train the existing images of the student and store the trained data in a xml file')
        elif(self.entry.get()=='face recognition'):
            self.text.insert(END,'\n'+'')
        elif(self.entry.get()=='photos'):
            self.text.insert(END,'\n'+'')
        elif(self.entry.get()=='attendance'):
            self.text.insert(END,'\n'+'')
        elif(self.entry.get()=='about us'):
            self.text.insert(END,'\n'+'')
        elif(self.entry.get()=='help desk'):
            self.text.insert(END,'\n'+'')
        elif(self.entry.get()=='exit'):
            self.text.insert(END,'\n'+'')
        elif(self.entry.get()=='attendance'):
            self.text.insert(END,'\n'+'')
        elif(self.entry.get()=='attendance'):
            self.text.insert(END,'\n'+'')

        else:
            self.text.insert(END,"\n"+"Bot: Sorry I didn't get it.Try something like 'student details','about us', ...")

# self.entry.get()==''

if __name__=='__main__':
    root=Tk()
    obj=ChatBot(root)
    root.mainloop()


from numpy import *
import numpy as np
from matplotlib import *
from scipy import *
from pylab import figure, show, setp
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
import scipy.io as sio
from scipy.signal import find_peaks

def step_rossler(x1,y1,z1,x2,y2,z2, dt, a, b, c, K,w1,w2):
    dx1 = dt*(-w1*y1-z1 + K*(x2-x1))
    dy1 = dt*(w1*x1 + a*y1)
    dz1 = dt*(b + z1*(x1 - c))

    dx2 = dt*(-w2*y2-z2 + K*(x1-x2))
    dy2 = dt*(w2*x2 + a*y2)
    dz2 = dt*(b + z2*(x2 - c))

    x1,y1,z1 = x1+dx1, y1+dy1, z1+dz1
    x2,y2,z2 = x2+dx2, y2+dy2, z2+dz2

    return (x1,y1,z1, x2,y2,z2)
def orbit_rossler(x1,y1,z1,x2,y2,z2, dt, a, b, c, K, w1,w2, numsteps):
    XYZ = np.zeros((numsteps+1)*6)
    XYZ = np.reshape(XYZ, (numsteps+1, 6))
    XYZ[0,:] = x1,y1,z1,x2,y2,z2
    for i in range(1, numsteps+1):
        x1,y1,z1,x2,y2,z2 = step_rossler(x1,y1,z1,x2,y2,z2, dt, a,b,c,K, w1, w2)

        XYZ[i,:] = x1,y1,z1,x2,y2,z2
    return XYZ
def simFn(x1,x2, skew):
    if skew == 0:
        diff_skew = x1 - x2
    else:
        diff_skew = x1[skew:] - x2[:-skew]

    diff_skew_avg = np.average(diff_skew*diff_skew)

    x1_sq_avg = np.average(x1*x1)
    x2_sq_avg = np.average(x2*x2)
    factor = np.sqrt(x1_sq_avg*x2_sq_avg)

    return diff_skew_avg/factor
x1_0, y1_0, z1_0 = [0.001]*3
x2_0, y2_0, z2_0 = [0.002]*3

a=0.165
b=0.2
c=10
w1=0.99
w2=0.95

dt = 0.01
t = np.arange(0,1000, dt)
numsteps = len(t) - 1
K = 0.2
XYZ_0_2 = orbit_rossler(x1_0, y1_0, z1_0, x2_0, y2_0, z2_0,dt,a,b,c, K, w1,w2, numsteps)
x1,y1,z1,x2,y2,z2 = [ XYZ_0_2[:,i] for i in range(6) ]
x1 = XYZ_0_2[:,0]
x2 = XYZ_0_2[:,3]

x1 = x1[10000:]
x2 = x2[10000:]

tau = np.arange(0,30,dt)
S = np.array([ simFn(x2,x1,int(_tau/dt)) for _tau in tau ])
minskew = np.argmin(S[:1000])
print(minskew)
#plt.plot(x1,x2)
#ax = plt.gca()
#ax.set_aspect(1.0)
#ax.set_xlabel('$x1(t)$')
#ax.set_ylabel('$x2(t)$')

#plt.plot(tau, S)
#ax = plt.gca()
#ax.set_xlabel('$\Delta t$')
#ax.set_ylabel('$S(\Delta t)$')

#plt.plot(x1,x2)
#ax = plt.gca()
#ax.set_aspect(1.0)
#ax.set_xlabel('$x1(t)$')
#ax.set_ylabel('$x2(t)$')

#plt.plot(tau, S)
#ax = plt.gca()
#ax.set_xlabel('$\Delta t$')
#ax.set_ylabel('$S(\Delta t)$')

plt.plot(x1[:-minskew], x2[minskew:])
ax = plt.gca()
ax.set_aspect(1.0)
ax.set_xlabel('$x1(t + \Delta t)$')
ax.set_ylabel('$x2(t)$')

show()

Картинки 17.py

Картинки 17.py


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import numpy as np
from scipy.io import loadmat
from sklearn import preprocessing  # 0-1编码
from sklearn.model_selection import StratifiedShuffleSplit  # 随机划分，保证每一类比例相同

class preprocess(object):

    def __init__(self):
        self.d_path='/Users/tung/Python/WorkProject/PHMresearch/WDCNN&LR_FaultDiagnosis/data/0HP'
        self.length=2048
        self.number=200
        self.normal=True
        self.rate=[0.5, 0.25, 0.25]
        self.enc = True
        self.enc_step = 28
        self.filenames = []

    def capture(self):
        """读取mat文件，返回字典

        :param original_path: 读取路径
        :return: 数据字典
        """
        files = {}
        for i in self.filenames:
            # 文件路径
            file_path = os.path.join(self.d_path, i)
            file = loadmat(file_path)
            file_keys = file.keys()
            for key in file_keys:
                if 'DE' in key:
                    files[i] = file[key].ravel()
        return files

    def slice_enc(self, data):
        """将数据切分为前面多少比例，后面多少比例.

        :param data: 单挑数据
        :param slice_rate: 验证集以及测试集所占的比例
        :return: 切分好的数据
        """
        slice_rate = self.rate[1] + self.rate[2]
        keys = data.keys()
        Train_Samples = {}
        Test_Samples = {}
        for i in keys:
            slice_data = data[i]
            all_lenght = len(slice_data)
            end_index = int(all_lenght * (1 - slice_rate))
            samp_train = int(self.number * (1 - slice_rate))  # 700
            Train_sample = []
            Test_Sample = []
            if self.enc:
                enc_time = self.length // self.enc_step
                samp_step = 0  # 用来计数Train采样次数
                for j in range(samp_train):
                    random_start = np.random.randint(low=0, high=(end_index - 2 * self.length))
                    label = 0
                    for h in range(enc_time):
                        samp_step += 1
                        random_start += self.enc_step
                        sample = slice_data[random_start: random_start + self.length]
                        Train_sample.append(sample)
                        if samp_step == samp_train:
                            label = 1
                            break
                    if label:
                        break
            else:
                for j in range(samp_train):
                    random_start = np.random.randint(low=0, high=(end_index - self.length))
                    sample = slice_data[random_start:random_start + self.length]
                    Train_sample.append(sample)

            # 抓取测试数据
            for h in range(self.number - samp_train):
                random_start = np.random.randint(low=end_index, high=(all_lenght - self.length))
                sample = slice_data[random_start:random_start + self.length]
                Test_Sample.append(sample)
            Train_Samples[i] = Train_sample
            Test_Samples[i] = Test_Sample
        return Train_Samples, Test_Samples

    # 仅抽样完成，打标签
    def add_labels(self, train_test):
        X = []
        Y = []
        label = 0
        for i in self.filenames:
            x = train_test[i]
            X += x
            lenx = len(x)
            Y += [label] * lenx
            label += 1
        return X, Y

    # one-hot编码
    def one_hot(self, Train_Y, Test_Y):
        Train_Y = np.array(Train_Y).reshape([-1, 1])
        Test_Y = np.array(Test_Y).reshape([-1, 1])
        Encoder = preprocessing.OneHotEncoder()
        Encoder.fit(Train_Y)
        Train_Y = Encoder.transform(Train_Y).toarray()
        Test_Y = Encoder.transform(Test_Y).toarray()
        Train_Y = np.asarray(Train_Y, dtype=np.int32)
        Test_Y = np.asarray(Test_Y, dtype=np.int32)
        return Train_Y, Test_Y

    def scalar_stand(self, Train_X, Test_X):
        # 用训练集标准差标准化训练集以及测试集
        scalar = preprocessing.StandardScaler().fit(Train_X)
        Train_X = scalar.transform(Train_X)
        Test_X = scalar.transform(Test_X)
        return Train_X, Test_X

    def valid_test_slice(self, Test_X, Test_Y):
        test_size = self.rate[2] / (self.rate[1] + self.rate[2])
        ss = StratifiedShuffleSplit(n_splits=1, test_size=test_size)
        for train_index, test_index in ss.split(Test_X, Test_Y):
            X_valid, X_test = Test_X[train_index], Test_X[test_index]
            Y_valid, Y_test = Test_Y[train_index], Test_Y[test_index]
            return X_valid, Y_valid, X_test, Y_test

    def prepro(self, d_path, length, number, normal, rate, enc, enc_step):
        self.d_path=d_path
        self.length=length
        self.number=number
        self.normal=normal
        self.rate=rate
        self.enc=enc
        self.enc_step=enc_step

        # 获得该文件夹下所有.mat文件名
        self.filenames = os.listdir(self.d_path)
        del self.filenames[2]   #删除.DS_Store

        # 从所有.mat文件中读取出数据的字典
        data = self.capture()
        # 将数据切分为训练集、测试集
        train, test = self.slice_enc(data)
        # 为训练集制作标签，返回X，Y
        Train_X, Train_Y = self.add_labels(train)
        # 为测试集制作标签，返回X，Y
        Test_X, Test_Y = self.add_labels(test)
        # 为训练集Y/测试集One-hot标签
        Train_Y, Test_Y = self.one_hot(Train_Y, Test_Y)
        # 训练数据/测试数据 是否标准化.
        if normal:
            Train_X, Test_X = self.scalar_stand(Train_X, Test_X)
        else:
            # 需要做一个数据转换，转换成np格式.
            Train_X = np.asarray(Train_X)
            Test_X = np.asarray(Test_X)
        # 将测试集切分为验证集合和测试集.
        Valid_X, Valid_Y, Test_X, Test_Y = self.valid_test_slice(Test_X, Test_Y)
        return Train_X, Train_Y, Valid_X, Valid_Y, Test_X, Test_Y

if __name__ == "__main__":

    """对数据进行预处理,返回train_X, train_Y, valid_X, valid_Y, test_X, test_Y样本.

    :param d_path: 源数据地址
    :param length: 信号长度，默认5个信号周期T=400*10S=66.6min～1h，2048
    :param number: 每种信号个数,总共9类,默认每个类别1000个数据
    :param normal: 是否标准化.True,Fales.默认True
    :param rate: 训练集/验证集/测试集比例.默认[0.5,0.25,0.25],相加要等于1
    :param enc: 训练集、验证集是否采用数据增强.Bool,默认True
    :param enc_step: 增强数据集采样顺延间隔
    :return: Train_X, Train_Y, Valid_X, Valid_Y, Test_X, Test_Y

    """
    #传感器研发组 做实验、售后去现场维护、采回来的数据
    #溶液地未连接(100)、流通池接地(100)、电缆线未连接(100)、球泡破裂(35)、支架损坏(31)、
    #电极污染(36）、电解液缺失(50)、水样波动(100)、正常
    #重采样之后 200*9 = 1800

    path = '/Users/tung/Python/WorkProject/PHMresearch/WDCNN&LR_FaultDiagnosis/data/0HP'
    test = preprocess()
    train_X, train_Y, valid_X, valid_Y, test_X, test_Y = test.prepro(d_path=path,
                                                                length=2048,
                                                                number=200,
                                                                normal=False,
                                                                rate=[0.7, 0.2, 0.1],
                                                                enc=False,
                                                                enc_step=28)
    print('训练样本维度:', train_X.shape)
    print(train_X.shape[0], '训练样本个数')
    print('训练标签one-hot维度',train_Y.shape[1])
    print('验证样本的维度', valid_X.shape)
    print(valid_X.shape[0], '验证样本个数')
    print('测试样本的维度', test_X.shape)
    print(test_X.shape[0], '测试样本个数')


import os
import string
import tempfile

from tensorflow.python.data.benchmarks import benchmark_base
from tensorflow.python.data.experimental.ops import readers
from tensorflow.python.data.ops import readers as core_readers
from tensorflow.python.ops import parsing_ops
from tensorflow.python.platform import gfile
from tensorflow.python.platform import googletest

class CsvDatasetBenchmark(benchmark_base.DatasetBenchmarkBase):
  """Benchmarks for `tf.data.experimental.CsvDataset`."""

  FLOAT_VAL = '1.23456E12'
  STR_VAL = string.ascii_letters * 10

  def _set_up(self, str_val):
    # Since this isn't test.TestCase, have to manually create a test dir
    gfile.MakeDirs(googletest.GetTempDir())
    self._temp_dir = tempfile.mkdtemp(dir=googletest.GetTempDir())

    self._num_cols = [4, 64, 256]
    self._num_per_iter = 5000
    self._filenames = []
    for n in self._num_cols:
      fn = os.path.join(self._temp_dir, 'file%d.csv' % n)
      with open(fn, 'w') as f:
        # Just write 100 rows and use `repeat`... Assumes the cost
        # of creating an iterator is not significant
        row = ','.join(str_val for _ in range(n))
        f.write('\n'.join(row for _ in range(100)))
      self._filenames.append(fn)

  def _tear_down(self):
    gfile.DeleteRecursively(self._temp_dir)

  def _run_benchmark(self, dataset, num_cols, prefix, benchmark_id):

    self.run_and_report_benchmark(
        dataset=dataset,
        num_elements=self._num_per_iter,
        name='%s_with_cols_%d' % (prefix, num_cols),
        iters=10,
        extras={
            'model_name': 'csv.benchmark.%d' % benchmark_id,
            'parameters': '%d' % num_cols,
        },
        warmup=True)

  def benchmark_map_with_floats(self):
    self._set_up(self.FLOAT_VAL)
    for i in range(len(self._filenames)):
      num_cols = self._num_cols[i]
      kwargs = {'record_defaults': [[0.0]] * num_cols}
      dataset = core_readers.TextLineDataset(self._filenames[i]).repeat()
      dataset = dataset.map(lambda l: parsing_ops.decode_csv(l, **kwargs))  # pylint: disable=cell-var-from-loop
      self._run_benchmark(
          dataset=dataset,
          num_cols=num_cols,
          prefix='csv_float_map_decode_csv',
          benchmark_id=1)
    self._tear_down()

  def benchmark_map_with_strings(self):
    self._set_up(self.STR_VAL)
    for i in range(len(self._filenames)):
      num_cols = self._num_cols[i]
      kwargs = {'record_defaults': [['']] * num_cols}
      dataset = core_readers.TextLineDataset(self._filenames[i]).repeat()
      dataset = dataset.map(lambda l: parsing_ops.decode_csv(l, **kwargs))  # pylint: disable=cell-var-from-loop
      self._run_benchmark(
          dataset=dataset,
          num_cols=num_cols,
          prefix='csv_strings_map_decode_csv',
          benchmark_id=2)
    self._tear_down()

  def benchmark_csv_dataset_with_floats(self):
    self._set_up(self.FLOAT_VAL)
    for i in range(len(self._filenames)):
      num_cols = self._num_cols[i]
      kwargs = {'record_defaults': [[0.0]] * num_cols}
      dataset = core_readers.TextLineDataset(self._filenames[i]).repeat()
      dataset = readers.CsvDataset(self._filenames[i], **kwargs).repeat()  # pylint: disable=cell-var-from-loop
      self._run_benchmark(
          dataset=dataset,
          num_cols=num_cols,
          prefix='csv_float_fused_dataset',
          benchmark_id=3)
    self._tear_down()

  def benchmark_csv_dataset_with_strings(self):
    self._set_up(self.STR_VAL)
    for i in range(len(self._filenames)):
      num_cols = self._num_cols[i]
      kwargs = {'record_defaults': [['']] * num_cols}
      dataset = core_readers.TextLineDataset(self._filenames[i]).repeat()
      dataset = readers.CsvDataset(self._filenames[i], **kwargs).repeat()  # pylint: disable=cell-var-from-loop
      self._run_benchmark(
          dataset=dataset,
          num_cols=num_cols,
          prefix='csv_strings_fused_dataset',
          benchmark_id=4)
    self._tear_down()

if __name__ == '__main__':
  benchmark_base.test.main()


from django.conf.urls.defaults import patterns

urlpatterns = patterns('channelguide.user_profile.views',
    (r'^register/?$', 'register_view'),
    (r'^search$', 'search'),
    (r'^moderators$', 'moderators'),
    (r'^confirm/(\d+)/([a-z0-9]+)', 'confirm'),
    (r'^set_language_view$', 'set_language_view'),
    (r'^profile/?$', 'user'),
    (r'^profile/(\d+)/$', 'user'),
)

urlpatterns += patterns('django.contrib.auth.views',
    (r'logout/?$', 'logout', {'next_page': '/'}))


### various utilities for plotting data
### for general analysis purposes
### mostly working with final data, some matlab arrays
import matplotlib
matplotlib.use('Agg') # to avoid auto-presenting images, breaking when no X server
import matplotlib.pyplot as plt

from scipy.ndimage import gaussian_filter
from .data_utils import trial_average

def plot_psths(data, fname_root, labels=None, sigma=20):
    """
    Plot PSTHs for each neuron in a final data list.
    fname_root is the root path the figures will be saved to
    labels are the unique label for each image (at the end of the
    fname_root, and in the title of the plots)
        - defaulting to the neuron indices
    """
    data = trial_average(data).mean(0) # across conditions too
    n_neurons = data.shape[0]
    if labels is None:
        labels = list(range(n_neurons))

    for cell in range(n_neurons):
        psth = gaussian_filter(data[cell], sigma=sigma) * 1000

        plt.figure()
        plt.plot(psth, linewidth=3)
        plt.xlabel('time (ms)')
        plt.ylabel('firing rate (Hz)')
        plt.title(f'PSTH for {labels[cell]}')
        plt.savefig(fname_root + labels[cell] + '.png')
        plt.close()


from KPAPI import sendMessage, typingoff
from config import config, load_json, save_json
from Action.All_Menu import teacher, student
import requests, random, json

url = "https://gist.githubusercontent.com/Nguyen3006-IT/85ef6863ddbaaecb3c39d490bf7802b3/raw/f0553d0f2880e245479fa58083098ed0faf79d8d/text2len.json"
json_text = requests.get(url).json()

def conjunction(sender_id, message_text):
  collect_text(message_text.lower())
  try:
    dic_text = load_json("accounts.json") or {}
    texts_collect = load_json("textNoiTu.json") or {}

    if dic_text[sender_id] == "noitu_empty":
      text_user = message_text.split()[-1]
      text_true = [text for text in json_text if text_user == text.split()[0]]
      if text_true == []:
        dic_text[sender_id] = "noitu_empty"
        sendMessage(sender_id, "Bot hết từ... Mong bạn cung cấp thêm giúp Bot.")
      else:
        dic_text[sender_id] = random.choice(text_true)
        sendMessage(sender_id, dic_text[sender_id])

    elif message_text.lower().split()[0] == dic_text[sender_id].lower().split()[-1] and len(message_text.split()) == 2:
      text_user = message_text.split()[-1]
      text_true = [text for text in json_text+texts_collect if text_user == text.split()[0]]
      if text_true == []:
        dic_text[sender_id] = "noitu_empty"
        sendMessage(sender_id, "Bot hết từ. Mời bạn đi trước.")
      else:
        dic_text[sender_id] = random.choice(text_true)
        sendMessage(sender_id, dic_text[sender_id])

    else:
      sendMessage(sender_id, "Bạn đã thua. Bot đi trước")
      dic_text[sender_id] = random.choice(json_text)
      sendMessage(sender_id, dic_text[sender_id])

    typingoff(sender_id)
    save_json("accounts.json", dic_text)

  except Exception as e:
        print(e)

def ongame(sender_id):
  dic_text = load_json("accounts.json")
  dic_text[sender_id] = json_text[random.randint(1,500)]
  save_json("accounts.json", dic_text)
  sendMessage(sender_id, dic_text[sender_id])

def offgame(sender_id):
  tool = load_json("tools.json")
  tool["online_noitu"].remove(sender_id)
  teacher(sender_id) if sender_id in tool["teacher"] else student(sender_id)
  save_json("tools.json", tool)
  sendMessage(sender_id, "Bạn đã rời trò chơi nối từ thành công.")

def collect_text(text):
  texts_collect = load_json("textNoiTu.json")
  texts_collect.append(text)
  with open("textNoiTu.json", 'w') as save:
    json.dump(texts_collect, save, ensure_ascii=False)

def rep_start_conjunction(sender_id):
  params = {"access_token": config["Token"]}
  headers = {"Content-Type": "application/json"}
  data = json.dumps({
      "recipient":{"id": sender_id},
      "message":{
        "attachment":{
        "type":"template",
        "payload":{
            "template_type":"button",
            "text": "Chọn người đi trước",
            "buttons":[{
              "type":"postback",
              "title": "SSBot",
              "payload":"bot_start"
            },
            {
              "type":"postback",
              "title": "Tôi",
              "payload": "user_start"
            }
            ]}}}})
  requests.post("https://graph.facebook.com/v2.6/me/messages", params=params,headers=headers, data=data)