#!/usr/bin/env python
# -*- encoding: utf-8 -*-

"""
Aperi'Solve - Flask application
Aperi'Solve is a web steganography plateform.

__author__ = "@Zeecka"
__copyright__ = "WTFPL"

"""

import os
from flask import Flask, render_template, request, jsonify, send_from_directory
from appfunct import getExt, cmdline
import stega

app = Flask(__name__)

APP_PORT = int(os.getenv('APP_PORT', 80))
APP_RM_FILE_TIME = int(os.getenv('APP_RM_FILE_TIME', 10))  # Keep images for ?m
app.config['MAX_CONTENT_LENGTH'] = int(
    os.getenv('APP_MAX_SIZE', 16 * 1024 * 1024))  # 16 Mega per image maxi
# Supported image types
imgExts = ["jpeg", "png", "bmp", "gif", "tiff", "jpg", "jfif", "jpe", "tif"]

@app.route('/')
def main():
    """ Upload Page """
    return render_template('index.html')

@app.errorhandler(Exception)
def error_handler(e):
    """ Error handler, mostly for wrong file """
    if "cannot identify" in str(e):  # Unknow image format
        # cannot identify image file 'uploads/xxx.jpg'
        a, *filename, b = str(e).split("'")  # Recover filename
        filename = ''.join(filename)
        os.remove(filename)  # Remove invalid file
        return jsonify({"Error": "Unknow file format."})
    return jsonify({"Error": str(e)})

@app.route('/upload', methods=['POST'])
def uploadImage():
    """ Upload image on server
    @fileup: Uploaded image (file)
    ---
    Return JSON array with:
    File: New file name on server
    Error: Error if upload fail
    """

    if "fileup" in request.files:
        f = request.files["fileup"]  # uploaded file
        ext = getExt(f.filename)

        if ext not in imgExts:
            return jsonify({"Error": "Invalid extension: "+str(ext)})

        newfilename = f.filename  # randString()+"."+ext
        newpathfile = f"uploads/{newfilename}"
        f.save(newpathfile)  # Save file with new name

        return jsonify({"File": newfilename})

    return jsonify({"Error": "No file submitted."})

@app.route('/process', methods=['POST'])
def process():
    """ Process layers decomposition
    @filename : Uploaded image (filename)
    ---
    Return JSON array with:
    Images : n * 8 images (path)
    Error: Error if file doesnt exist
    """
    if "filename" in request.form:
        f = request.form["filename"]  # already uploaded file
        pathfile = f"uploads/{f}"

        if not os.path.isfile(pathfile):
            return jsonify({"Error": "File doesn't exist."})

        images = stega.processImage(f, "uploads/")  # Generate Images

        return jsonify({"Images": images})
    return jsonify({"Error": "No filename submitted."})

@app.route('/zsteg', methods=['POST'])
def zsteg():
    """ Process zsteg analysis.
    @filename : Uploaded image (filename)
    @allzsteg : -a option for zsteg (bool)
    @zstegfiles : extract option for zsteg (bool)
    Return JSON array with:
    Zsteg : zsteg output
    Error: Error if file doesnt exist
    """
    if "filename" in request.form:
        f = request.form["filename"]  # already uploaded file
        pathfile = f"uploads/{f}"

        if not os.path.isfile(pathfile):
            return jsonify({"Error": "File doesn't exist."})

        allzsteg, zstegfiles = False, False
        if "allzsteg" in request.form:
            allzsteg = bool(int(request.form["allzsteg"]))
        if "zstegfiles" in request.form:
            zstegfiles = bool(int(request.form["zstegfiles"]))

        zstegoutput = stega.processZsteg(f, "uploads/", allzsteg,
                                         zstegfiles)

        return jsonify({"Zsteg": zstegoutput})
    return jsonify({"Error": "No filename submitted."})

@app.route('/binwalk', methods=['POST'])
def binwalk():
    """ Process binwalk analysis.
    @filename : Uploaded image (filename)
    Return JSON array with:
    Binwalk : binwalk output
    Error: Error if file doesnt exist
    """
    if "filename" in request.form:
        f = request.form["filename"]  # already uploaded file
        pathfile = f"uploads/{f}"

        if not os.path.isfile(pathfile):
            return jsonify({"Error": "File doesn't exist."})

        binwalkoutput = stega.processBinwalk(f, "uploads/")

        return jsonify({"Binwalk": binwalkoutput})
    return jsonify({"Error": "No filename submitted."})

@app.route('/foremost', methods=['POST'])
def foremost():
    """ Process foremost analysis.
    @filename : Uploaded image (filename)
    Return JSON array with:
    Binwalk : binwalk output
    Error: Error if file doesnt exist
    """
    if "filename" in request.form:
        f = request.form["filename"]  # already uploaded file
        pathfile = f"uploads/{f}"

        if not os.path.isfile(pathfile):
            return jsonify({"Error": "File doesn't exist."})

        foremostoutput = stega.processForemost(f, "uploads/")

        return jsonify({"Foremost": foremostoutput})
    return jsonify({"Error": "No filename submitted."})

@app.route('/steghide', methods=['POST'])
def steghide():
    """ Process steghide analysis.
    @filename : Uploaded image (filename)
    @passwdsteg : Steghide passwd
    Return JSON array with:
    Steghide : Steghide output
    Error: Error if file doesnt exist / password is wrong
    """
    if "filename" in request.form:
        f = request.form["filename"]  # already uploaded file
        pathfile = f"uploads/{f}"

        if not os.path.isfile(pathfile):
            return jsonify({"Error": "File doesn't exist."})

        if len(request.form["passwdsteg"]):
            steghideoutput = stega.processSteghide(
                f, "uploads/", request.form["passwdsteg"])
        else:
            steghideoutput = {"Error":
                              "Steghide doesn't work without password."}

        return jsonify({"Steghide": steghideoutput})
    return jsonify({"Error": "No filename submitted."})

@app.route('/outguess', methods=['POST'])
def outguess():
    """ Process outguess analysis.
    @filename : Uploaded image (filename)
    @passwdsteg : Outguess passwd
    Return JSON array with:
    Steghide : Outguess output
    Error: Error if file doesnt exist / password is wrong
    """
    if "filename" in request.form:
        f = request.form["filename"]  # already uploaded file
        pathfile = f"uploads/{f}"

        if not os.path.isfile(pathfile):
            return jsonify({"Error": "File doesn't exist."})

        outguessoutput = stega.processOutguess(
            f, "uploads/", request.form["passwdsteg"])

        return jsonify({"Outguess": outguessoutput})
    return jsonify({"Error": "No filename submitted."})

@app.route('/exiftool', methods=['POST'])
def exiftool():
    """ Process exiftools analysis.
    @filename : Uploaded image (filename)
    Return JSON array with:
    Exiftool : Exiftool output
    Error: Error if file doesnt exist
    """
    if "filename" in request.form:
        f = request.form["filename"]  # already uploaded file
        pathfile = f"uploads/{f}"

        if not os.path.isfile(pathfile):
            return jsonify({"Error": "File doesn't exist."})

        exiftooloutput = stega.processExif(f, "uploads/")

        return jsonify({"Exiftool": exiftooloutput})
    return jsonify({"Error": "No filename submitted."})

@app.route('/strings', methods=['POST'])
def strings():
    """ Process strings analysis.
    @filename : Uploaded image (filename)
    Return JSON array with:
    Strings : Strings output
    Error: Error if file doesnt exist
    """
    if "filename" in request.form:
        f = request.form["filename"]  # already uploaded file
        pathfile = f"uploads/{f}"

        if not os.path.isfile(pathfile):
            return jsonify({"Error": "File doesn't exist."})

        stringsimg = stega.processStrings(f, "uploads/")

        return jsonify({"Strings": stringsimg})
    return jsonify({"Error": "No filename submitted."})

@app.route('/uploads/<path>')
def uploads(path):
    """ Route for uploaded/computed files
    Remove old uploaded/computed files on each requests
    """

    # First remove old files
    cmdline(f"find uploads/ -mmin +{APP_RM_FILE_TIME} "
            r"-type f  \( -iname \"*\" ! -iname \".gitkeep\" \) "
            r"-exec rm -fv {} \;")
    return send_from_directory('uploads', path)

if __name__ == '__main__':
    app.run(debug=True, host='0.0.0.0', port=APP_PORT)


from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.common.action_chains import ActionChains # <--- helpful!
from selenium.webdriver.common.by import By
import configparser

from bs4 import BeautifulSoup

import pandas as pd
from pandas.io.json import json_normalize

import os
from time import sleep
#import tqdm
import sys

print('-'*60)

# Using one example image
# url = 'https://www.instagram.com/p/CGsyLsUhk0K/'# <-- no trailing slash on url
url = sys.argv[1] # <-- Second argument of the dockerfile CMD (first argument is the python script name itself!)

if not url.startswith('https://'):
    url = 'https://' + url

# Raw HTML file outdir
raw_outfile = "/data/raw/{}.html".format(url.split('/')[-1]) # <--- outdir html file
print('Raw html outfile prepared as: {}'.format(raw_outfile))

# ETL (extracted from raw HTML) file outdir
etl_outfile = "/data/etl/{}.json".format(url.split('/')[-1]) # <--- outdir json file
print('ETL outfile prepared as: {}'.format(etl_outfile))

# Instagram HTML element
xpath_aria_label_button = 'Load more comments' # <--- inspect & find class name for button click

# Optimization parameters
threshold_for_failed_click = 5  # <-- Int
time_to_wait_for_nxt_click = 0.25 # <-- Int

##############################################
# Setup Selenium Driver

# driver = webdriver.Chrome(executable_path='/Users/etiennejacquot/Downloads/chromedriver_v84')
# driver = webdriver.Firefox()  # <-- desktop app firefox

options = webdriver.ChromeOptions() # <-- for selenium-chrome
#options = webdriver.FirefoxOptions() # for selnium-firefox
selenium_url = 'http://localhost:4444/wd/hub' # <-- docker standalonee selenium browser
options.add_argument('--headless')
driver = webdriver.Remote(command_executor=selenium_url,desired_capabilities=options.to_capabilities())

driver = webdriver.Remote(command_executor=selenium_url,desired_capabilities=options.to_capabilities())

# Navigate to instagram post url
driver.get(url)

i = 0 # <--- USED FOR CLICK COUNT
j = 0 # <--- THRESHOLD FOR FAILED CLICKED

while j < threshold_for_failed_click:
    #print('J value is --> {}'.format(j))
    ##############################################
    # Sleep and give browser a chance to refresh
    sleep(time_to_wait_for_nxt_click) # <--- Time to wait until next click

    ##############################################
    # Try to find button to load more comments
    try:
        element = driver.find_element(By.XPATH, '//button[./span[@aria-label="{}"]]'.format(xpath_aria_label_button))
    except:
        #print("\nUnexpected error:", sys.exc_info()[0])
        j = j+1 # <--- Plus one for failed click
        continue

    ##############################################
    # Try to click button to load more comments
    print('Trying to click & load more comments --> {}'.format(i))

    try:
        ActionChains(driver).move_to_element(element).click().perform()
    except:
        #print("\nUnexpected error:", sys.exc_info()[0])
        j=j+1 # <--- Plus one for failed click
        continue

    i=i+1 # <--- Successful cycle for clicked account, +1
    j = 0 # <--- FAILED CLICK COUNTER RESET

    if i > 100:
        print('oops! clicked 100 times apparently... not sure why this is happening so exiting this container to continue onward!')
        j=threshold_for_failed_click+1

##############################################
# Exporting HTML raw data

with open (raw_outfile,"w+") as f:
    print('Exporting to .html file --> {}'.format(raw_outfile))
    f.write(driver.page_source)

##############################################
# ETL conversion
#driver.quit() # close out of chrome tab (or browser?)
driver.close() # close out of chrome tab (or browser?)

comment_df = pd.DataFrame()

with open(raw_outfile, "r") as f:
    contents = f.read()
    soup = BeautifulSoup(contents, 'lxml')

# Comments are ul objects in HTML
comments_html = soup.find_all('ul')

# Loop through comments in html file to extract ETL
for x in comments_html:

    a = x.find('div')

    # get author name
    try:
        author = a.get_text(separator=' ').split(' ')[0] # <--- No spaces in Instagram usernames!
    except:
        author = None
        pass

    # get comment on post
    try:
        comment = a.get_text(separator=' ').split(' ')[1:]
        comment = " ".join(comment)
    except:
        comment = None
        pass

    comment_df = comment_df.append(pd.DataFrame({'url':[url],'author':[author],'comment':[comment]}))

print('Done! Number of comments extracted --> {}'.format(comment_df.shape[0]))
print('Exporting ETL json file --> {}'.format(etl_outfile))

comment_df.reset_index(drop=True).to_json('{}'.format(etl_outfile))

print('Complete!')
print('-'*75)


import networkx as nx

class UndirectedGraph:
    def __init__(self, graph_data):
        self.graph = nx.Graph()
        self.graph_data = graph_data[1:]
        self.cardinality = int(graph_data[0])
        self.graph_type = 'Undirected Graph'

    def get_nodes(self):
        [self.graph.add_node(node) for node in range(self.cardinality)]
        return self.graph.number_of_nodes(), self.graph.nodes()

    def get_edges(self):
        for x in range(self.cardinality):
            for y in range(self.cardinality):
                if self.graph_data[x][y] == '1':
                    self.graph.add_edge(x, y)
        return self.graph.number_of_edges(), self.graph.edges()

    def get_degrees(self):
        return nx.degree(self.graph)

    def get_graph_type(self):
        return self.graph_type

    def get_adjacency_matrix(self):
        for x in range(self.cardinality):
            print(' ', format(x, '>1'), end=' ')
            if x == self.cardinality - 1:
                print('')
        for y in range(self.cardinality):
            print(y, '   '.join(n for n in self.graph_data[y]))

    def display(self):
        print('')
        print('Type: ', self.get_graph_type())
        print('Node Cardinality: %s\nNodes: %s' % self.get_nodes())
        print('Edge Cardinality: %s\nEdges %s' % self.get_edges())
        print('Degrees: ', self.get_degrees())
        print('Adjacency Matrix: ')
        self.get_adjacency_matrix()


import numpy as np
import pandas as pd
import datetime as dt
import matplotlib.pyplot as plt
import matplotlib.dates as date
import seaborn as sns
import urllib

sns.set_context('talk')

data_crime_raw = pd.read_csv('.\\NYPD_Complaint_Data_Historic.csv',
                             usecols=['CMPLNT_FR_DT', 'OFNS_DESC', 'LAW_CAT_CD', 'Latitude', 'Longitude'],
                             dtype={'OFNS_DESC':'category', 'LAW_CAT_CD':'category', 'Latitude':float, 'Longitude':float})

data_crime_raw['CMPLNT_FR_DT'] = pd.to_datetime(data_crime_raw['CMPLNT_FR_DT'], format='%m/%d/%Y', errors='coerce')

data_311_raw = pd.read_csv('.\\311_Service_Requests_from_2010_to_Present.csv',
                           usecols=['Created Date', 'Complaint Type', 'Descriptor', 'Latitude', 'Longitude'],
                           dtype={'Complaint Type':'category', 'Descriptor':'category', 'Latitude':float, 'Longitude':float})

data_311_raw['created_date'] = pd.to_datetime(data_311_raw['Created Date'], format='%m/%d/%Y %I:%M:%S %p', errors='coerce')

data_crime = data_crime_raw[data_crime_raw.CMPLNT_FR_DT > pd.to_datetime(dt.date(2010,1,1))].dropna()
data_311 = data_311_raw[data_311_raw.created_date < pd.to_datetime(dt.date(2016,1,1))].dropna()

minlat = data_crime.Latitude.min()
maxlat = data_crime.Latitude.max()
minlon = data_crime.Longitude.min()
maxlon = data_crime.Longitude.max()

latrange = np.arange(minlat, maxlat+0.02, 0.02)
lonrange = np.arange(minlon, maxlon+0.02, 0.02)

data_crime = data_crime[data_crime.LAW_CAT_CD != 'VIOLATION']

d_c_grouped = data_crime.groupby(
    by=[pd.cut(data_crime['Latitude'], latrange),
        pd.cut(data_crime['Longitude'], lonrange),
        pd.TimeGrouper(key='CMPLNT_FR_DT',freq='M')])

d_311_grouped = data_311.groupby(
    by=[pd.cut(data_311['Latitude'], latrange),
        pd.cut(data_311['Longitude'], lonrange),
        pd.TimeGrouper(key='created_date',freq='M')])

max_var_loc = d_c_grouped.size().unstack().var(axis=1).argmax()

data_crime_window = data_crime_raw[data_crime_raw.CMPLNT_FR_DT.between(
    pd.to_datetime(dt.date(2010,1,1)), pd.to_datetime(dt.date(2015,2,1)))].dropna()
data_311_window = data_311_raw[data_311_raw.created_date.between(
    pd.to_datetime(dt.date(2010,1,1)), pd.to_datetime(dt.date(2015,2,1)))].dropna()

d_c_win_grouped = data_crime_window.groupby(
    by=[pd.cut(data_crime_window['Latitude'], latrange),
        pd.cut(data_crime_window['Longitude'], lonrange),
        pd.TimeGrouper(key='CMPLNT_FR_DT',freq='5D')])

d_311_win_grouped = data_311_window.groupby(
    by=[pd.cut(data_311_window['Latitude'], latrange),
        pd.cut(data_311_window['Longitude'], lonrange),
        pd.TimeGrouper(key='created_date',freq='5D')])

plt.close('all')
fig, ax = plt.subplots(figsize=(20,10))

d_c_plot = d_c_win_grouped.size().unstack().loc[max_var_loc][:-1]
crime_regression = np.polyfit(d_c_plot.index.astype(np.int64), d_c_plot.data, 1)
plot_crime = ax.plot_date(d_c_plot.index, d_c_plot.data, 'b', label='NYPD complaints')
reg_crime = ax.plot_date(d_c_plot.index, crime_regression[0]*d_c_plot.index.astype(int)+crime_regression[1], 'r',
                        label='NYPD complaints linear regression')

d_311_plot = d_311_win_grouped.size().unstack().loc[max_var_loc][:-1]
regression = np.polyfit(d_311_plot.index.astype(np.int64), d_311_plot.data, 1)
plot_311 = ax.plot_date(d_311_plot.index, d_311_plot.data, 'g', label='311 complaints')
reg_311 = ax.plot_date(d_311_plot.index, regression[0]*d_311_plot.index.astype(int)+regression[1], 'm',
                      label='311 complaints linear regression')

ax.legend()

plt.savefig('NYPDand311trend.png', format='png')

plt.close('all')
fig, ax = plt.subplots(figsize=(20, 10))

d_311_largest = d_311_grouped['Complaint Type'].value_counts().unstack().loc[max_var_loc].sum().nlargest(30)
d_311_select = data_311_window[data_311_window['Complaint Type'].isin(d_311_largest.index)]

d_311_select_grouped = d_311_select.groupby(
    by=[pd.cut(d_311_select['Latitude'], latrange),
        pd.cut(d_311_select['Longitude'], lonrange),
        pd.TimeGrouper(key='created_date',freq='M')])

d_311_select_plot = d_311_select_grouped['Complaint Type'].value_counts().unstack().loc[max_var_loc][:-1]
d_311_select_plot.plot(kind='bar', stacked=True, ax=ax)
ax.legend(loc='center left', bbox_to_anchor=(1.0, 0.5))
xtl=[dt.datetime.strptime(item.get_text(), '%Y-%m-%d %H:%M:%S').strftime('%b %Y') for item in ax.get_xticklabels()]
_=ax.set_xticklabels(xtl)
fig.subplots_adjust(right=0.8)
plt.savefig('complaints311categories.png', format='png')


"""A JSON-RPC request object. Used internally by the library, but class
attributes can be modified to configure various options for handling requests.
"""

import json
import logging
import traceback
import re
import pkgutil
try:
    # Python 2
    from collections import Mapping, Sequence
except ImportError:
    # Python 3
    from collections.abc import Mapping, Sequence

from funcsigs import signature
import jsonschema

from jsonrpcserver import config
from jsonrpcserver.log import _log
from jsonrpcserver.response import RequestResponse, NotificationResponse, \
    ExceptionResponse
from jsonrpcserver.exceptions import JsonRpcServerError, InvalidRequest, \
    InvalidParams, MethodNotFound

logger = logging.getLogger(__name__)

json_validator = jsonschema.Draft4Validator(json.loads(pkgutil.get_data(
    __name__, 'request-schema.json').decode()))

def _get_method(methods, name):
    """Finds a method in a list (or dictionary).

    :param methods: List or dictionary of named functions.
    :param name: Name of the method to find.
    :raises MethodNotFound: If the method wasn't in the list.
    :returns: The method from the list.
    """
    # If it's a Mapping (dict-like), search for the key
    if isinstance(methods, Mapping):
        try:
            return methods[name]
        except KeyError:
            raise MethodNotFound(name)
    # Otherwise it must be a Sequence (list-like), search the __name__
    # attributes
    elif isinstance(methods, Sequence):
        try:
            return next(m for m in methods if m.__name__ == name)
        except StopIteration:
            raise MethodNotFound(name)

def _convert_camel_case(name):
    """Convert a camelCase string to under_score"""
    s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
    return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()

def _convert_camel_case_keys(original_dict):
    """Converts all keys of a dict from camelCase to under_score, recursively"""
    new_dict = dict()
    for k, v in original_dict.items():
        if isinstance(v, dict):
            # Recurse
            new_dict[_convert_camel_case(k)] = _convert_camel_case_keys(v)
        else:
            new_dict[_convert_camel_case(k)] = v
    return new_dict

def _validate_against_schema(request):
    """Validate against the JSON-RPC schema.

    :param request: JSON-RPC request dict.
    :raises InvalidRequest: If the request is invalid.
    :returns: None
    """
    try:
        json_validator.validate(request)
    except jsonschema.ValidationError as e:
        raise InvalidRequest(e.message)

def _validate_arguments_against_signature(func, args, kwargs):
    """Check if arguments match a function signature and can therefore be passed
    to it.

    :param func: The function object.
    :param args: List of positional arguments (or None).
    :param kwargs: Dict of keyword arguments (or None).
    :raises InvalidParams: If the arguments cannot be passed to the function.
    """
    try:
        if not args and not kwargs:
            signature(func).bind()
        elif args:
            signature(func).bind(*args)
        elif kwargs:
            signature(func).bind(**kwargs)
    except TypeError as e:
        raise InvalidParams(str(e))

def _call(methods, method_name, args=None, kwargs=None):
    """Find a method from a list, then validate the arguments before calling it.

    :param methods: The list of methods - either a python list, or Methods obj.
    :param args: Positional arguments (list)
    :param kwargs: Keyword arguments (dict)
    :raises MethodNotFound: If the method is not in the list.
    :raises InvalidParams: If the arguments don't match the method signature.
    :returns: The return value from the method called.
    """
    # Get the method object from a list of rpc methods
    method = _get_method(methods, method_name)
    # Ensure the arguments match the method's signature
    _validate_arguments_against_signature(method, args, kwargs)
    # Call the method
    if args and kwargs:
        # Cannot have both positional and keyword arguments in JSON-RPC.
        raise InvalidParams()
    # No arguments
    elif not args and not kwargs:
        return method()
    # Positional arguments
    elif args:
        return method(*args)
    # Keyword arguments
    elif kwargs:
        return method(**kwargs)

def _get_arguments(request):
    """Takes the 'params' part of a JSON-RPC request and converts it to either
    positional or keyword arguments usable in Python. The value can be a JSON
    array (python list), object (python dict), or omitted. There are no other
    acceptable options. Note that a JSON-RPC request can have positional or
    keyword arguments, but not both! See
    http://www.jsonrpc.org/specification#parameter_structures

    :param request: JSON-RPC request in dict form.
    :raises InvalidParams: If 'params' was present but was not a list or dict.
    :returns: A tuple containing the positionals (in a list, or None) and
    keywords (in a dict, or None) extracted from the 'params' part of the
    request.
    """
    positionals = keywords = None
    params = request.get('params')
    # Params was omitted from the request. Taken as no arguments.
    if 'params' not in request:
        pass
    # Params is a list. Taken as positional arguments.
    elif isinstance(params, list):
        positionals = params
    # Params is a dict. Taken as keyword arguments.
    elif isinstance(params, dict):
        keywords = params
    # Anything else is invalid. (This should never happen if the request has
    # passed the schema validation.)
    else:
        raise InvalidParams('Params of type %s is not allowed' % \
            type(params).__name__)
    return (positionals, keywords)

class Request(object):
    """JSON-RPC Request object.

    Encapsulates a JSON-RPC request, providing details such as the method name,
    arguments, and whether it's a request or a notification, and provides a
    ``process`` method to execute the request.
    """

    def __init__(self, request):
        """
        :param request: JSON-RPC request, in dict or string form
        """
        # Validate against the JSON-RPC schema
        if config.schema_validation:
            _validate_against_schema(request)
        # Get method name from the request. We can assume the key exists because
        # the request passed the schema.
        self.method_name = request['method']
        # Get arguments from the request, if any
        self.args, self.kwargs = _get_arguments(request)
        # Get request id, if any
        self.request_id = request.get('id')
        # Convert camelCase to underscore
        if config.convert_camel_case:
            self.method_name = _convert_camel_case(self.method_name)
            if self.kwargs:
                self.kwargs = _convert_camel_case_keys(self.kwargs)

    @property
    def is_notification(self):
        """Returns True if the request is a JSON-RPC notification (ie. No
        response is required, False if it's a request.
        """
        return self.request_id is None

    def process(self, methods):
        """Calls the method and returns a Response object."""
        error = None
        try:
            result = _call(methods, self.method_name, self.args, self.kwargs)
        # Catch any JsonRpcServerError raised (Invalid Request, etc)
        except JsonRpcServerError as e:
            error = e
        # Catch uncaught exceptions and respond with ServerError
        except Exception as e: # pylint: disable=broad-except
            # Log the uncaught exception
            _log(logger, 'error', traceback.format_exc())
            error = e # pylint: disable=redefined-variable-type
        if error:
            if self.is_notification and not config.notification_errors:
                return NotificationResponse()
            else:
                return ExceptionResponse(error, self.request_id)
        # Success
        if self.is_notification:
            return NotificationResponse()
        else:
            return RequestResponse(self.request_id, result)


import logging
from tkinter import ttk, TclError

from hoverset.ui.icons import get_icon_image, get_icon
from hoverset.ui.widgets import Canvas, FontStyle, Frame, Entry, Button, Label, ScrollableInterface, EventMask
from studio.ui import geometry

class CollapseFrame(Frame):

    def __init__(self, master, **cnf):
        super().__init__(master, **cnf)
        self.config(**self.style.dark)
        self._label_frame = Frame(self, **self.style.bright, height=20)
        self._label_frame.pack(side="top", fill="x", padx=2)
        self._label_frame.pack_propagate(0)
        self._label = Label(self._label_frame, **self.style.bright, **self.style.text_bright)
        self._label.pack(side="left")
        self._collapse_btn = Button(self._label_frame, width=20, **self.style.bright, **self.style.text_bright)
        self._collapse_btn.config(text=get_icon("triangle_up"))
        self._collapse_btn.pack(side="right", fill="y")
        self._collapse_btn.on_click(self.toggle)
        self.body = Frame(self, **self.style.dark)
        self.body.pack(side="top", fill="both", pady=2)
        self.__ref = Frame(self.body, height=0, width=0, **self.style.dark)
        self.__ref.pack(side="top")
        self._collapsed = False

    def update_state(self):
        self.__ref.pack(side="top")

    def collapse(self, *_):
        if not self._collapsed:
            self.body.pack_forget()
            self._collapse_btn.config(text=get_icon("triangle_down"))
            self.pack_propagate(0)
            self.config(height=20)
            self._collapsed = True

    def clear_children(self):
        self.body.clear_children()

    def expand(self, *_):
        if self._collapsed:
            self.body.pack(side="top", fill="both")
            self.pack_propagate(1)
            self._collapse_btn.config(text=get_icon("triangle_up"))
            self._collapsed = False

    def toggle(self, *_):
        if self._collapsed:
            self.expand()
        else:
            self.collapse()

    @property
    def label(self):
        return self._label["text"]

    @label.setter
    def label(self, value):
        self._label.config(text=value)

class SideBar(Canvas):

    def __init__(self, master):
        super().__init__(master)
        self.config(**self.style.dark, **self.style.no_highlight, width=20)
        self.features = {}

    def remove(self, feature):
        self.delete(feature.indicator)
        self.features.pop(feature)
        self._redraw()

    def _redraw(self):
        y = 0
        for feature in self.features:
            indicator = self.features[feature]
            font = FontStyle(self, self.itemconfig(indicator).get("font", "TkDefaultFont")[3])
            y += font.measure(feature.name) + 20
            self.coords(indicator, 18, y)

    def add_feature(self, feature):
        indicator = self.create_text(0, 0, angle=90, text=feature.name, fill=self.style.dark_on_hover.get("background"),
                                     anchor="sw", activefill=self.style.dark_on_hover.get("background"))
        font = FontStyle(self, self.itemconfig(indicator).get("font", "TkDefaultFont")[3])
        y = font.measure(feature.name) + self.bbox("all")[3] + 20
        self.coords(indicator, 18, y)
        self.tag_bind(indicator, "<Button-1>", lambda event: self.toggle_feature(feature))
        feature.indicator = indicator
        self.features[feature] = indicator

    def change_feature(self, new, old):
        self.tag_unbind(old.indicator, "<Button-1>")
        self.tag_bind(old.indicator, "<Button-1>", lambda event: self.toggle_feature(new))
        self.features.pop(old)
        self.features[new] = old.indicator
        new.indicator = old.indicator

    def deselect(self, feature):
        self.itemconfig(feature.indicator, fill=self.style.dark_text.get("foreground"))

    def select(self, feature):
        self.itemconfig(feature.indicator, fill=self.style.dark_on_hover.get("background"))

    def close_all(self):
        for feature in self.features:
            self.deselect(feature)

    def toggle_feature(self, feature):
        feature.toggle()

class SearchBar(Frame):

    def __init__(self, master=None, **cnf):
        super().__init__(master, **cnf)
        self.config(**self.style.no_highlight, **self.style.dark)
        self._entry = Entry(self, **self.style.dark_input)
        self._clear_btn = Button(self, image=get_icon_image("close", 15, 15),
                                 **self.style.dark_button, width=25, height=25)
        self._clear_btn.pack(side="right", fill="y")
        self._clear_btn.on_click(self._clear)
        Label(self, **self.style.dark_text, image=get_icon_image("search", 15, 15)).pack(side="left")
        self._entry.pack(side="left", fill="both", expand=True, padx=2)
        self._entry.on_entry(self._change)
        self._on_change = None
        self._on_clear = None

    def focus_set(self):
        super().focus_set()
        self._entry.focus_set()

    def on_query_change(self, func, *args, **kwargs):
        self._on_change = lambda val: func(val, *args, **kwargs)

    def on_query_clear(self, func, *args, **kwargs):
        self._on_clear = lambda: func(*args, **kwargs)

    def _clear(self, *_):
        if self._on_clear:
            self._on_clear()

    def _change(self, *_):
        if self._on_change:
            self._on_change(self._entry.get())

class DesignPad(ScrollableInterface, Frame):
    PADDING = 10

    def __init__(self, master=None, **kwargs):
        super().__init__(master, **kwargs)
        self._frame = Canvas(self, **kwargs, **self.style.no_highlight, **self.style.dark)
        self._frame.grid(row=0, column=0, sticky='nswe')
        self._scroll_y = ttk.Scrollbar(master, orient='vertical', command=self._y_scroll)
        self._scroll_x = ttk.Scrollbar(master, orient='horizontal', command=self._x_scroll)
        self._frame.configure(yscrollcommand=self._scroll_y.set, xscrollcommand=self._scroll_x.set)
        self.columnconfigure(0, weight=1)  # Ensure the design_pad gets the rest of the left horizontal space
        self.rowconfigure(0, weight=1)
        self._frame.bind('<Configure>', self.on_configure)
        self.bind('<Configure>', self.on_configure)
        self._child_map = {}
        self._on_scroll = None
        # Ensure scroll_region always begins at 0, 0
        # To achieve this we need a line/point at position 2, 2 with the minimum possible width of 1
        # Without this workaround positioning on the design pad becomes unstable
        self._frame.create_line(2, 2, 2, 2)

    def on_mousewheel(self, event):
        try:
            if event.state & EventMask.CONTROL and self._scroll_x.winfo_ismapped():
                self._frame.xview_scroll(-1 * int(event.delta / 50), "units")
            elif self._scroll_y.winfo_ismapped():
                self._frame.yview_scroll(-1 * int(event.delta / 50), "units")
        except TclError:
            pass

    def scroll_position(self):
        return self._scroll_y.get()

    def on_scroll(self, callback, *args, **kwargs):
        self._on_scroll = lambda: callback(*args, **kwargs)

    def _y_scroll(self, *args):
        self._frame.yview(*args)
        if self._on_scroll:
            self._on_scroll()

    def _x_scroll(self, *args):
        self._frame.xview(*args)
        if self._on_scroll:
            self._on_scroll()

    def _show_y_scroll(self, flag):
        if flag and not self._scroll_y.winfo_ismapped():
            self._scroll_y.grid(in_=self, row=0, column=1, sticky='ns')
        elif not flag:
            self._scroll_y.grid_forget()
        self.update_idletasks()

    def _show_x_scroll(self, flag):
        if flag and not self._scroll_x.winfo_ismapped():
            self._scroll_x.grid(in_=self, row=1, column=0, columnspan=2, sticky='ew')
        elif not flag:
            self._scroll_x.grid_forget()
        self.update_idletasks()

    def on_configure(self, *_):
        try:
            self.update_idletasks()
            scroll_region = self._frame.bbox('all')
        except TclError:
            return
        if not scroll_region:
            logging.error("failed to acquire scroll region")
            return

        scroll_w = scroll_region[2] - scroll_region[0]
        scroll_h = scroll_region[3] - scroll_region[1]

        self._show_y_scroll(scroll_h > self.winfo_height())
        self._show_x_scroll(scroll_w > self.winfo_width())

        self._frame.config(scrollregion=scroll_region)

    def place_child(self, child, **kw):
        x = kw.get("x", 0)
        y = kw.get("y", 0)
        w = kw.get("width", 1)
        h = kw.get("height", 1)
        self.forget_child(child)
        if child in self._child_map:
            self.config_child(child, **kw)
        else:
            window = self._frame.create_window(x, y, window=child, width=w, height=h, anchor='nw')
            self._child_map[child] = window
        self.on_configure()

    def bbox(self, child):
        # return the canvas bbox if possible else use the normal bound
        # canvas bbox is more accurate
        if self._child_map.get(child is not None):
            return self._frame.bbox(self._child_map[child])
        return geometry.relative_bounds(geometry.bounds(child), self._frame)

    def config_child(self, child, **kw):
        x1, y1, x2, y2 = self.bbox(child)
        x = kw.get("x", x1)
        y = kw.get("y", y1)
        w = kw.get("width", x2 - x1)
        h = kw.get("height", y2 - y1)
        if not kw:
            return {
                "x": x, "y": y, "width": w, "height": h
            }
        self._frame.coords(self._child_map[child], x, y)
        self._frame.itemconfigure(self._child_map[child], width=w, height=h)
        self.on_configure()

    def forget_child(self, child):
        if self._child_map.get(child) is not None:
            self._frame.delete(self._child_map[child])
            self._child_map.pop(child)

    def configure(self, cnf=None, **kw):
        self._frame.configure(cnf, **kw)
        return super().configure(cnf, **kw)

    config = configure


import numpy as np
import matplotlib.pyplot as plt
import bisect

# Base class for all noisy signals
# Not to be used directly. Use the derived classes instead
#
class NoiseSignal(object):
    """
    Generic noisy signal
    """

    def __init__(self, initial_seed=1234, resolution=1.0, time_unit=1e-9):
        self.resolution_factor = resolution
        self.amplitude = 1.0
        self.random_seed = initial_seed
        self.is_init = False
        self.interpolation_settings()
        self.time_unit=time_unit
        self.time_units_per_sec = 1/time_unit

        # This base random seed is used to create the individual seeds used to generate each instance of the noise
        self.seed_generator = np.random.RandomState(self.random_seed)

    def __repr__(self):
        return "NoiseSignal(resolution_factor={0}, amplitude={1}, seed={2}".format(self.resolution_factor, self.amplitude, self.random_seed)
    def __str__(self):
        return "NoiseSignal(resolution_factor={0}, amplitude={1}, seed={2}".format(self.resolution_factor, self.amplitude, self.random_seed)
    def __getitem__(self, key):
        if self.do_interpolation:
            return self.eval_interpolated(key)
        else:
            return self.eval(key)

    def reset(self):
        self.seed_generator = np.random.RandomState(self.random_seed)

    def get_new_seed(self):
        return self.seed_generator.randint(np.iinfo(np.int32).max)

    def interpolation_settings(self, do_interpolation=False, resolution_factor=1.0, itime=10.0):
        # enable flag
        self.do_interpolation = do_interpolation
        # resolution of samples from the signal
        self.interpolation_resolution = resolution_factor
        # window to interpolate over
        self.interpolation_time = itime

    def eval_interpolated(self, t):
        if self.do_interpolation:
            time_steps, data_samples = self.get_sample_data(t)
            ival = np.interp(t, time_steps, data_samples)*self.amplitude
            return ival
        else:
            return self.eval(t)

class NoiseSignalRandomWalk(NoiseSignal):
    '''
    Creates a random walk function, starting at 0, over a specified time interval.
    Create the object, configure it, then init. Do more inits or next_intervals as necessary
    '''
    def __str__(self):
        return "NoiseSignalRandomWalk(resolution_factor={0}, amplitude={1}, seed={2}".format(self.resolution_factor, self.amplitude, self.random_seed)

    def configure_noise(self, amplitude, resolution_factor, total_time):
        '''total_time will remain constant after configuration, but the times and points will get data appended to
        each time you call init or next_interval.
        amplitude: the step size of each walk; the value at a given time is the last time's value plus or minus this amplitude
        resolution: how many values per time_unit
        '''
        self.amplitude = amplitude
        self.resolution_factor = resolution_factor
        self.total_time = total_time #number of time_units per .init() call
        self.points = []
        self.times = []
        self.num_points = int(self.total_time*self.resolution_factor)
        self.next_interval_start_time = 0

    def init(self, random_seed=None):
        '''
        Starts a random walk from zero, up to total_time with num_points in between.
        '''
        if random_seed==None:
            random_seed = self.get_new_seed()
        rand_switch = np.random.RandomState(random_seed)
        num_points = self.num_points
        binaries = [0] + [self.amplitude if rand_switch.randint(2)==1 else -1*self.amplitude for i in range(num_points-1)]
        points = [0]*num_points
        points[0] = 0
        for i in range(1, num_points):
            points[i] = points[i-1] + binaries[i]
        self.points += points
        step = (self.total_time - 0)/self.num_points
        if len(self.times) == 0:
            self.times += [i*step for i in range(num_points)]  #array of times from 0 to total_time, units of time_unit
            self.next_interval_start_time = num_points*step
        else:
            self.times += [self.next_interval_start_time+i*step for i in range(num_points)]
            self.next_interval_start_time = self.next_interval_start_time + num_points*step
        self.is_init = True
        return random_seed

    def eval(self, t):
        '''Returns noise value at time t (specified in time_unit) in units of time_unit'''
        if t>self.total_time:
            raise ValueError("NoiseSignalNormal(): Requested time greater than total simulation time")
        if t<0:
            raise ValueError("NoiseSignalNormal(): Requested time < 0")

        tidx = int(t*self.resolution_factor)
        return self.points[tidx]

    def get_sample_data(self):
        '''returns a tuple with a time list and associated noise values at each time point'''
        times = self.times #returns all the calculated times, including any .next_intervals() you may do
        vals = self.points
        return times, vals

    def next_interval(self,random_seed=None):
        '''Does self.init again, but starts where the last init ended, not from zero.
        Will will add total_time amount of time to self.time (a total of num_points added)
        '''
        if random_seed==None:
            random_seed = self.get_new_seed()
        rand_switch = np.random.RandomState(random_seed)
        num_points = int(self.total_time * self.resolution_factor)
        binaries = [self.amplitude if rand_switch.randint(2)==1 else -1*self.amplitude for i in range(num_points)]
        points = [0]*num_points
        points[0] = binaries[0] + self.points[len(self.points) -1]
        for i in range(1, num_points):
            points[i] = points[i-1] + binaries[i]
        self.points += points
        step = (self.total_time - 0)/self.num_points
        self.times += [self.next_interval_start_time+i*step for i in range(num_points)]
        self.next_interval_start_time = self.next_interval_start_time + num_points*step
        return random_seed

    def plot_noise_signal(self):
        if self.is_init == False:
            raise ValueError("Must be initialized first!")
        marker = '.'
        linewidth = 1
        markersize = 1

        t, n = self.get_sample_data()
        plt.plot(t,n, marker=marker, linewidth=linewidth, markersize=markersize)
        plt.grid()
        plt.title("Random Walk Noise\n$\pm${} per step, {:.1f}(x{:.1e}s) per step".format(self.amplitude, self.total_time/self.num_points, self.time_unit) )
        plt.ylabel("Arbitrary Units")
        plt.xlabel("Time, in units of {:.1e} seconds".format(self.time_unit))
        plt.show()

class NoiseSignalSine(NoiseSignal):
    '''
    Creates a noise signal composed of a summation of sine waves over a specified time interval.
    create the object, configure it, then do an init and you're good to go. Do a next_interval or another init if necessary
    '''
    def __str__(self):
        return "NoiseSignalSine(resolution_factor={0}, seed={1}".format(self.resolution_factor, self.random_seed)

    def configure_noise(self, resolution_factor, freq_list, amp_list, phase_list, total_time):
        '''
        resolution factor: how many points per time_unit
        freq_list: a list or tuple of the all the desired frequencies, in Hz
        amp_list: a list or tuple of the amplitudes for all frequencies
        phase_list: a list or tuple of the phases of all the waves (all lists should be the same length)
        total_time: total # of time_units you want to cover per .init call
        '''
        self.resolution_factor = resolution_factor
        self.total_time = total_time #number of time_units per .init() call
        self.list_of_point_lists = []
        self.summed_points = []
        self.times = []
        self.num_points = int(self.total_time*self.resolution_factor)
        self.step = self.total_time/self.num_points #the time interval between time points, in units of time_units
        self.next_interval_start_time = 0
        self.freq_list = freq_list
        self.amp_list = amp_list
        self.phase_list = phase_list

    def init(self):
        times = [i*self.step for i in range(0,self.num_points+1)] #step = totaltime/numpoints, numpoits = int(totaltime*resfactor)
        #print(times)
        list_of_point_lists = [0]*len(self.freq_list)
        for i in range(len(self.freq_list)):
            '''find values for each frequency, then sum them up later'''
            freq = self.freq_list[i]
            amp = self.amp_list[i]
            phase = self.phase_list[i]
            value_list = []
            for t in times:
                val = amp*np.sin(2*np.pi*freq*t*self.time_unit + phase)
                value_list.append(val)
            list_of_point_lists[i] = value_list
        self.list_of_point_lists = list_of_point_lists
        #each frequency's list is done, now add them all together in one list
        summed_vals = [0]*len(times)
        for t_index in range(len(times)):
            for wave_index in range(len(self.list_of_point_lists)):
                summed_vals[t_index] += self.list_of_point_lists[wave_index][t_index]
        #print(summed_vals)
        self.summed_points += summed_vals
        #add these lines at the end of the init call
        self.times += times
        self.next_interval_start_time = self.times[-1] + self.step

    def get_sample_data(self):
        '''returns a tuple with a time list and associated noise values at each time point'''
        times = self.times #returns all the calculated times, including any .next_intervals() you may do
        vals = self.summed_points #returns the summed sine wave for each point in self.times
        return times, vals

    def add_random_noise(self, sigma):
        '''add normally distributed noise to the overall summed_values
        Specify the std deviation, sigma, of the noise, which centers on zero'''
        for i in range(len(self.summed_points)):
            self.summed_points[i] += np.random.normal(0,sigma)

    def next_interval(self):
        times = [self.next_interval_start_time+i*self.step for i in range(self.num_points)]
        #print(times)
        for i in range(len(self.freq_list)):
            '''find values for each frequency, then su mthem up later'''
            freq = self.freq_list[i]
            amp = self.amp_list[i]
            phase = self.phase_list[i]
            value_list = []
            for t in times:
                val = amp*np.sin(2*np.pi*freq*t*self.time_unit + phase)
                value_list.append(val)
            self.list_of_point_lists[i].extend(value_list)

        #each frequency's list is done, now add them all together in one list
        summed_vals = [0]*len(times)
        for t_index in range(len(times)):
            for wave_index in range(len(self.list_of_point_lists)):
                summed_vals[t_index] += self.list_of_point_lists[wave_index][t_index]
        #print(summed_vals)
        self.summed_points += summed_vals
        #add these lines at the end of the init call
        self.times += times
        self.next_interval_start_time = self.times[-1] + self.step
        #print("After next interval, end time is {}".format(self.times[-1]))

    def plot_noise_signal(self, show_components=True):
        '''Shows a plot of the individual waves, and a plot of the summed wave, if components = True'''

        times = self.times
        summed = self.summed_points
        components = self.list_of_point_lists
        marker ='.'
        linewidth=1
        markersize = 1
        if show_components == True:
            for wave_index in range(len(components)):
                wave= components[wave_index]
                freq = self.freq_list[wave_index]
                plt.plot(times, wave, label="{:.2f} Hz".format(freq), marker=marker, linewidth=linewidth, markersize=markersize)
            plt.grid()
            plt.xlabel("Time in selected units ({:.0e} s)".format(self.time_unit))
            plt.ylabel("Amplitude, a.u.")
            plt.legend()
            plt.title("Component Waves")
            plt.show()

        plt.plot(times, summed, marker=marker, linewidth=linewidth,markersize=markersize)
        plt.grid()
        plt.xlabel("Time in selected units ({:.0e} s)".format(self.time_unit))
        plt.ylabel("Amplitude, a.u.")
        plt.title("Sinusoidal Noise Signal")
        plt.show()

    def eval(self, t):
        if t>self.total_time:
            print("Bad time is {}".format(t))
            raise ValueError("NoiseSignalSine(): Requested time greater than total simulation time")
        if t<0:
            raise ValueError("NoiseSignalSine(): Requested time < 0")

        tidx = int(t*self.resolution_factor)
        #print("time t is {}".format(t))
        #print("tindex is {}".format(tidx))
        #print("total time is {}".format(self.total_time))
        return self.summed_points[tidx]

class NoiseSignalNormal(NoiseSignal):

    def __str__(self):
        return "NoiseSignalNormal(resolution_factor={0}, amplitude={1}, seed={2}, mean{3}, variance{4})".format(self.resolution_factor, self.amplitude, self.random_seed, self.mean, self.variance)
    def __repr__(self):
        if self.is_init:
            str_val = "normal_deviants = [ "
            for ii in range(len(self.points)):
                str_val += "{0} ".format(self.points[ii])
            str_val += "]"

            return str_val
        else:
            return "normal_deviants = []"

    def plot_amplitudes(self):
        times = [ti/self.resolution_factor for ti in range(int(self.total_time*self.resolution_factor))]
        plt.plot(times, [sv*self.amplitude for sv in self.points])
        plt.show()

    def configure_noise(self, amplitude, mean, variance, resolution_factor, total_time):
        self.mean = mean
        self.variance = variance
        self.amplitude = amplitude
        self.resolution_factor = resolution_factor
        self.std_dev = np.sqrt(variance)
        self.total_time = total_time

    def init(self, random_seed=None):
        if random_seed==None:
            random_seed = self.get_new_seed()
        rand_switch = np.random.RandomState(random_seed)
        num_points = int(self.total_time * self.resolution_factor)

        self.points = [self.std_dev*rand_switch.randn() + self.mean for ii in range(num_points)]
        self.is_init = True

        return random_seed

    def eval(self, t):
        if t>self.total_time:
            raise ValueError("NoiseSignalNormal(): Requested time greater than total simulation time")
        if t<0:
            raise ValueError("NoiseSignalNormal(): Requested time < 0")

        tidx = int(t*self.resolution_factor)
        return self.points[tidx]*self.amplitude

    def get_sample_data(self, t):
        lidx = int(np.floor(t*self.resolution_factor))
        ltime = lidx/self.resolution_factor
        uidx = int(np.ceil(t*self.resolution_factor))
        utime = uidx/self.resolution_factor

        sample_times = [ltime]
        sample_vals = [self.points[lidx]]
        if uidx<len(self.points):
            sample_times += [utime]
            sample_vals += [self.points[uidx]]

        while (lidx>0 or (uidx+1)<len(self.points)) and ((len(sample_times)<2) or (utime-ltime)<self.interpolation_time):
            if lidx>0:
                lidx -= 1
                ltime = lidx/self.resolution_factor
                sample_vals = [self.points[lidx]] + sample_vals
                sample_times = [ltime] + sample_times
            if (uidx+1)<len(self.points):
                uidx += 1
                utime = uidx/self.resolution_factor
                sample_vals += [self.points[uidx]]
                sample_times += [utime]

        if len(sample_times)<2:
            raise ValueError("Cannot find enough sample data")

        return sample_times, sample_vals

class NoiseSignalTelegraph(NoiseSignal):

    def __str__(self):
        return "NoiseSignalTelegraph(resolution_factor={0}, amplitude={1}, seed={2}, exponent={3}, total_fluctuators={4}, start_freq={5}, stop_freq={6}, total_time={7})".format(self.resolution_factor, self.amplitude, self.random_seed, self.exponent, self.total_fluctuators, self.start_freq, self.stop_freq, self.total_time)
    def __repr__(self):
        if self.is_init:
            str_val = "flux_switch_times = [ "
            for ii in range(len(self.switching_times)):
                str_val += "{0}:{1} ".format(self.switching_times[ii],self.amplitude_sums[ii])
            str_val += "]"
        else:
            return "flux_switch_times = []"

        return str_val

    def plot_amplitudes(self):
        prev_time = -1
        switch_times = []
        switch_amps = []
        for ii in range(len(self.switching_times)):
            sw_time = self.switching_times[ii]
            sw_sum = self.amplitude*self.amplitude_sums[ii]

            if prev_time >= 0:
                switch_times += [sw_time]
                switch_amps += [prev_amp]
            switch_times += [sw_time]
            switch_amps += [sw_sum]
            prev_time = sw_time
            prev_amp = sw_sum

        plt.plot(switch_times, switch_amps)
        plt.show()

    def plot_noise_signal(self):
        if self.is_init != True:
            raise ValueError("Noise signal must first be initialized!")
        num_points = int(self.total_time/self.resolution_factor)
        step = self.total_time/num_points
        times = [i*step for i in range(num_points)] #step = totaltime/numpoints, numpoits = int(totaltime*resfactor)
        vals = []
        for t in times:
            vals.append(self[t])
        plt.plot(times, vals,marker='.',linewidth=1)
        plt.xlabel("Time in time_unit")
        plt.ylabel("Noise, arb. units")
        plt.title("Telegraph Noise\nFrom {} Hz to {} Hz".format(self.start_freq, self.stop_freq))
        plt.grid()
        plt.show()

    def configure_noise(self, exponent, amplitude, total_fluctuators, start_freq, stop_freq, total_time):
        self.exponent = float(exponent)
        self.amplitude = float(amplitude / (2.0 * np.pi))
        self.total_fluctuators = total_fluctuators
        self.start_freq = float(start_freq)
        self.stop_freq = float(stop_freq)
        self.total_time = float(total_time)
        self.create_fluctuators()

    # create structures needed for noise generator
    def create_fluctuators(self):

        freq_area = np.log(self.stop_freq/self.start_freq)/((self.total_fluctuators-1))
        freq_fact = np.exp(freq_area)

        if ( self.exponent<0.0 or self.exponent>2.0 ):
            raise ValueError("Invalid exponent for Telegraph noise, should be > 0.0 and <= 2.0")

        self.freq_points = [self.start_freq*np.power(freq_fact,ii) for ii in range(self.total_fluctuators+1)]
        if self.exponent==1.0:
            self.freq_amps = [np.sqrt(2.0 * (np.log(self.freq_points[ii+1])-np.log(self.freq_points[ii]))) for ii in range(self.total_fluctuators)]
        else:
            self.freq_amps = [np.sqrt(1.0/(1.0-self.exponent) * (np.power(self.freq_points[ii+1],1.0-self.exponent) - np.power(self.freq_points[ii],1.0-self.exponent))) for ii in range(self.total_fluctuators)]
        del self.freq_points[self.total_fluctuators]

    # initialize a new set of fluctuators
    def init(self, random_seed=None):
        if random_seed==None:
            random_seed = self.get_new_seed()
        self.rand_switch = np.random.RandomState(random_seed)

        # initialize the state of each fluctuator (+-1.0)
        self.f_state = [1.0 if self.rand_switch.randint(2)==0 else -1.0 for ii in range(self.total_fluctuators)]

        # initialize the switching time for each fluctuator
        # next switching time is exponentially distributed, so we can
        # generate a uniform rv and transform it to an exponential rv.
        # also convert from seconds to nanoseconds (1 s = 10^9 ns) and start at a random point in the interval [0,1]
        self.f_next_switch = [(self.time_units_per_sec)*self.rand_switch.uniform()*(-np.log(1-self.rand_switch.uniform())/self.freq_points[ii]) for ii in range(self.total_fluctuators)]

        # create an array of switching times (for any fluctuator) and the sum of all the fluctuators at that time
        #
        self.amplitude_sums = [sum(self.f_state[ii]*self.freq_amps[ii] for ii in range(self.total_fluctuators))]
        self.switching_times = [0.0]

        tval = 0.0;
        while ( tval < self.total_time ):
            # find the fluctuator that is to switch next
            next_switch_idx = self.f_next_switch.index(min(self.f_next_switch))
            tval = self.f_next_switch[next_switch_idx]

            # update the switching time for this fluctuator
            self.f_next_switch[next_switch_idx] += self.time_units_per_sec*(-np.log(1-self.rand_switch.uniform())/self.freq_points[next_switch_idx])

            # update the state of this fluctuator
            self.f_state[next_switch_idx] *= -1.0

            # add the switching time and new sum to the lists
            self.switching_times += [tval]
            self.amplitude_sums += [self.amplitude_sums[len(self.amplitude_sums)-1] + self.f_state[next_switch_idx]*2.0*self.freq_amps[next_switch_idx]]

        self.is_init = True
        return random_seed

    # move to a new interval of time, i.e., continue the 1/f signal without re-initializing
    def next_interval(self):

        self.f_next_switch = [next_switch-self.total_time for next_switch in self.f_next_switch]

        self.amplitude_sums = [sum(self.f_state[ii]*self.freq_amps[ii] for ii in range(self.total_fluctuators))]
        self.switching_times = [0.0]

        tval = 0.0;
        while ( tval < self.total_time):
            # find the fluctuator that is to switch next
            next_switch_idx = self.f_next_switch.index(min(self.f_next_switch))
            tval = self.f_next_switch[next_switch_idx]

            # update the switching time for this fluctuator
            self.f_next_switch[next_switch_idx] += self.time_units_per_sec*(-np.log(1-self.rand_switch.uniform())/self.freq_points[next_switch_idx])

            # update the state of this fluctuator
            self.f_state[next_switch_idx] *= -1.0

            # add the switching time and new sum to the lists
            self.switching_times += [tval]
            self.amplitude_sums += [self.amplitude_sums[len(self.amplitude_sums)-1] + self.f_state[next_switch_idx]*2.0*self.freq_amps[next_switch_idx]]

    # return the value of the noise at the indicated time
    def eval(self, t):
        tdx = bisect.bisect_left(self.switching_times, t)
        if ( tdx >= len(self.switching_times) ):
             tdx -= 1

        if tdx==0:
            e_val = self.amplitude_sums[0]*self.amplitude
        else:
            e_val = self.amplitude_sums[tdx-1]*self.amplitude
        return e_val

    def get_sample_data(self, t):
        uidx = 0
        while uidx<len(self.switching_times) and self.switching_times[uidx]<t:
            uidx += 1

        if uidx>0:
            lidx = uidx - 1
        else:
            lidx = 0

        ltime = self.switching_times[lidx]
        utime = self.switching_times[uidx]
        sample_times = [ltime]
        sample_vals = [self.amplitude_sums[lidx]]
        if not ltime == utime:
            sample_times += [utime]
            sample_vals += [self.amplitude_sums[uidx]]

        while (lidx>0 or (uidx+1)<len(self.switching_times)) and ((len(sample_times)<2) or (utime-ltime)<self.interpolation_time):
            if lidx>0:
                lidx -= 1
                ltime = self.switching_times[lidx]
                sample_vals = [self.amplitude_sums[lidx]] + sample_vals
                sample_times = [ltime] + sample_times
            if (uidx+1)<len(self.switching_times):
                uidx += 1
                utime = self.switching_times[uidx]
                sample_vals += [self.amplitude_sums[uidx]]
                sample_times += [utime]

        if len(sample_times)<2:
            raise ValueError("Cannot find enough sample data")

        return sample_times, sample_vals

if __name__=='__main__':
    total_time = 1000
    time_unit = 1e-3

    sig = NoiseSignalTelegraph(initial_seed=1234, time_unit=time_unit, resolution=1)
    sig.configure_noise(exponent=1, amplitude=1, total_fluctuators=40, start_freq=0.1, stop_freq=5, total_time=total_time)
    sig.init()
    sig.plot_noise_signal()

    amp = 1
    res = 1
    freq_list = (2,5, 8) #Hz
    amp_list = (0.1, 0.06, 0.04) #arb. units
    phase_list = (0,0,0)
    sig = NoiseSignalRandomWalk(initial_seed = 1, time_unit=time_unit)
    sig.configure_noise(amp, res, total_time)
    sig.init()
    sig.plot_noise_signal()

    sine = NoiseSignalSine(time_unit = 1e-3)
    sine.configure_noise(resolution_factor=0.1, freq_list=freq_list, amp_list=amp_list, phase_list=phase_list, total_time=1000)
    sine.init()
    sine.next_interval()
    sine.plot_noise_signal(True)


import datetime

def print_header():
    title = 'BIRTHDAY APP'

    print('-' * 80)
    print(' ' * int((80 - len(title)) / 2) + title)
    print('-' * 80)

def get_users_birthday():
    print('Please tell me your birthday.')

    year = int(input('Year (YYYY): '))
    month = int(input('Month (M[M]): '))
    day = int(input('Day (D[D]): '))

    return datetime.date(year, month, day)

def get_days_until_next_birthday(birthday):
    now = datetime.date.today()
    birthday = birthday.replace(year=now.year)

    if birthday == now:
        return 0

    if birthday < now:
        birthday = birthday.replace(year=birthday.year + 1)

    return round((birthday - now).total_seconds() / 60 / 60 / 24)

def main():
    print_header()

    birthday = get_users_birthday()

    days = get_days_until_next_birthday(birthday)

    if days == 0:
        print('Happy birthday!')
    elif days == 1:
        print('Your birthday is tomorrow!')
    else:
        print('Your birthday is in {} days!'.format(days))

main()


from __future__ import print_function
import sys
import logging
import codecs
import argparse
from pyjade.utils import process
import os

def convert_file():
    support_compilers_list = ['django', 'jinja', 'underscore', 'mako', 'tornado', 'html']
    available_compilers = {}
    for i in support_compilers_list:
        try:
            if i == "django":
                from django.conf import settings
                settings.configure()

            compiler_class = __import__('pyjade.ext.%s' % i, fromlist=['pyjade']).Compiler
        except ImportError as e:
            logging.warning(e)
        else:
            available_compilers[i] = compiler_class

    usage = "usage: %(prog)s [options] [input [output]]"
    parser = argparse.ArgumentParser(usage=usage)

    parser.add_argument("-o", "--output", help="write output to FILE", metavar="FILE")

    # use a default compiler here to sidestep making a particular
    # compiler absolutely necessary (ex. django)
    default_compiler = sorted(available_compilers.keys())[0]
    parser.add_argument("-c", "--compiler",
                    choices=list(available_compilers.keys()),
                    default=default_compiler,
                    metavar="COMPILER",
                    help="template compiler, %s by default; supported compilers are: %s" % (default_compiler, ", ".join(available_compilers.keys())))
    parser.add_argument("-e", "--ext", dest="extension",
                      help="set import/extends default file extension",
                      metavar="FILE")
    parser.add_argument("--block_start_string", metavar="STRING", default="{%", help="valid for jinja compiler only, defaut is '{%%'")
    parser.add_argument("--block_end_string", metavar="STRING", default="%}", help="valid for jinja compiler only, defaut is '%%}'")
    parser.add_argument("--variable_start_string", metavar="STRING", default="{{", help="valid for jinja compiler only, defaut is '{{'")
    parser.add_argument("--variable_end_string", metavar="STRING", default="}}", help="valid for jinja compiler only, defaut is '}}'")
    parser.add_argument("input", nargs="?")
    parser.add_argument("output", nargs="?", default=argparse.SUPPRESS)

    args = parser.parse_args()

    compiler = args.compiler

    if args.extension:
        extension = '.%s' % args.extension
    elif args.output:
        extension = os.path.splitext(args.output)[1]
    else:
        extension = None

    if compiler in available_compilers:
        import six
        if args.input:
            template = codecs.open(args.input, 'r', encoding='utf-8').read()
        elif six.PY3:
            template = sys.stdin.read()
        else:
            template = codecs.getreader('utf-8')(sys.stdin).read()

        output = process(template, compiler=available_compilers[compiler],
                         staticAttrs=True, extension=extension,
                         block_start_string=args.block_start_string,
                         block_end_string=args.block_end_string,
                         variable_start_string=args.variable_start_string,
                         variable_end_string=args.variable_end_string)

        if args.output:
            outfile = codecs.open(args.output, 'w', encoding='utf-8')
            outfile.write(output)
        elif six.PY3:
            sys.stdout.write(output)
        else:
            codecs.getwriter('utf-8')(sys.stdout).write(output)
    else:
        raise Exception('You must have %s installed!' % compiler)

if __name__ == '__main__':
    convert_file()


from collections import defaultdict,deque
import functools
N,M = list(map(int,input().split()))
AB = [list(map(lambda x:int(x)-1,input().split())) for _ in range(M) ]
Q= int(input())

vdc = [ list(map(int,input().split())) for _ in range(Q)]
for vdc_elements in vdc:
    vdc_elements[0] = vdc_elements[0]-1
near_dict = defaultdict(set)

for a,b in AB:
        near_dict[a].add(b)
        near_dict[b].add(a)

@functools.lru_cache()
def get_distance(a,b):
    used = set()
    q = deque()
    q.append((a,0))
    while q :
        #print(used)
        now,dist=q.pop()
        if now ==b:
            return dist
        for i in near_dict[now]:
           if i not in used:
                q.append((i,dist+1))
                used.add(i)

    return 10000000000

def get_color(target_i):
    for i in reversed(range(Q)):
        v,d,c = vdc[i]
        if get_distance(v,target_i) <= d:
            return c
    return 0

#print(get_distance(0,3))
#print(get_distance(1,6))
if N+M+Q > 7000:
    raise
for i in range(N):
    print(get_color(i))


# uncompyle6 version 3.7.4
# Python bytecode 3.6 (3379)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /tmp/pip-build-ed191__6/protobuf/google/protobuf/internal/import_test_package/inner_pb2.py
# Compiled at: 2020-01-10 16:25:26
# Size of source mod 2**32: 2139 bytes
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(name='google/protobuf/internal/import_test_package/inner.proto',
  package='google.protobuf.python.internal.import_test_package',
  syntax='proto2',
  serialized_options=None,
  serialized_pb=b'\n8google/protobuf/internal/import_test_package/inner.proto\x123google.protobuf.python.internal.import_test_package"\x1a\n\x05Inner\x12\x11\n\x05value\x18\x01 \x01(\x05:\x0257')
_INNER = _descriptor.Descriptor(name='Inner',
  full_name='google.protobuf.python.internal.import_test_package.Inner',
  filename=None,
  file=DESCRIPTOR,
  containing_type=None,
  fields=[
 _descriptor.FieldDescriptor(name='value',
   full_name='google.protobuf.python.internal.import_test_package.Inner.value',
   index=0,
   number=1,
   type=5,
   cpp_type=1,
   label=1,
   has_default_value=True,
   default_value=57,
   message_type=None,
   enum_type=None,
   containing_type=None,
   is_extension=False,
   extension_scope=None,
   serialized_options=None,
   file=DESCRIPTOR)],
  extensions=[],
  nested_types=[],
  enum_types=[],
  serialized_options=None,
  is_extendable=False,
  syntax='proto2',
  extension_ranges=[],
  oneofs=[],
  serialized_start=113,
  serialized_end=139)
DESCRIPTOR.message_types_by_name['Inner'] = _INNER
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
Inner = _reflection.GeneratedProtocolMessageType('Inner', (_message.Message,), {'DESCRIPTOR':_INNER,
 '__module__':'google.protobuf.internal.import_test_package.inner_pb2'})
_sym_db.RegisterMessage(Inner)


import cv2
import os
from tqdm import tqdm
'''
Watch the videos via opencv, and save every <interval> frame.
'''
source_path = '/home/tk/Desktop/20190919record'
video_list = []
for root, dirs, files in os.walk(source_path):
    for i in files:
        if i.endswith('.dav'):
            video_list.append(os.path.join(root,i))

for videopath in tqdm(video_list):
    video_size = os.path.getsize(videopath)/(1000*1000)
    if video_size > 600:
        interval = 400
    else:
        interval = 200
    outputpath = os.path.join('data',videopath.split('/')[-1][:-4])
    try:
        os.mkdir(outputpath)
    except:
        pass
    cap = cv2.VideoCapture(videopath)
    time = 0
    while True:
        ret, frame = cap.read()
        if ret:
            time += 1
            if "PTZ" not in videopath:
                frame = frame[::-1,:,:]
                h,w = frame.shape[:2]
                sections = []
                for row in range(2):
                    for col in range(4):
                        sections.append(frame[row*(h//2):row*(h//2)+h//2,col*(w//4):col*(w//4)+w//4,:])

                for i in range(8):
                    cv2.namedWindow('frame_{}'.format(i), 0)
                    cv2.resizeWindow('frame_{}'.format(i), 502, 450);
                    cv2.imshow('frame_{}'.format(i), sections[i])
            #     if time%25==0:
            #         cv2.imwrite('data/videoframe/{}_{:d}.jpg'.format(time,i), sections[i])
            else:
                cv2.imshow('frame', frame)

            if time%interval ==0:
                cv2.imwrite(os.path.join(outputpath,'{:d}.jpg'.format(time//interval)), frame)
            if cv2.waitKey(1) & 0xFF == ord('q'):
                break
        else:
            break
    print('Captures of video {} has been saved in {}'.format(videopath.split('/')[-1],outputpath))

cap.release()
cv2.destroyAllWindows()


from __future__ import print_function
import torchvision
import torchvision.transforms as transforms
import os
import time
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import matplotlib.pyplot as plt

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(device)

num_classes = 10
batch_size  = 100
learning_rate = 1e-3
num_epochs = 15*6 # max epoch

thresh = 0.1 # neuronal threshold (V)
lens = thresh # hyper-parameters of approximate function
delta_t = 2**(-10) #s
tau1, tau2 = 40e-3, 10e-3 #s
# delta_t = 1 # ms
# tau1, tau2 = 40, 10 # ms
const = tau1 / (tau1 - tau2)

max_firing_rate = 200 # hz
# max_firing_rate = 0.1

decay1 = np.exp(-delta_t/tau1)
decay2 = np.exp(-delta_t/tau2)

# define approximate firing function
class ActFun(torch.autograd.Function):

    @staticmethod
    def forward(ctx, input):
        ctx.save_for_backward(input)
        return input.gt(thresh).float()

    @staticmethod
    def backward(ctx, grad_output):
        input, = ctx.saved_tensors
        grad_input = grad_output.clone()
        temp = abs(input - thresh) < lens
        return grad_input * temp.float()

act_fun = ActFun.apply

# membrane potential update
def mem_update(ops, x, mem1, mem2, spike):
    mem1 = mem1 * decay1 * (1. - spike) + const * ops(x)
    mem2 = mem2 * decay2 * (1. - spike) + const * ops(x)
    mem = mem1 - mem2
    spike = act_fun(mem) # act_fun : approximation firing function
    return mem1, mem2, spike

# Dacay learning_rate
def lr_scheduler(optimizer, epoch, init_lr=0.1, lr_decay_epoch=50):
    """Decay learning rate by a factor of 0.1 every lr_decay_epoch epochs."""
    if epoch % lr_decay_epoch == 0 and epoch > 1:
        for param_group in optimizer.param_groups:
            # param_group['lr'] = param_group['lr'] * 0.1
            param_group['lr'] = param_group['lr'] * 0.5
    return optimizer

class SCNN(nn.Module):
    def __init__(self):
        super(SCNN, self).__init__()
        self.conv1 = nn.Conv2d(1, 16, 4, stride=2, bias=False)
        self.conv2 = nn.Conv2d(16, 32, 3, bias=False)
        self.conv3 = nn.Conv2d(32, 64, 3, stride=2, bias=False)
        self.conv4 = nn.Conv2d(64, 10, 4, bias=False)
        self.fc1 = nn.Linear(40, 10, bias=False)

    def forward(self, input, time_window = 128):
        h1_mem1 = h1_mem2 = h1_spike = torch.reshape(torch.zeros(batch_size, 13*13*16, device=device), (batch_size,-1,13,13))
        h2_mem1 = h2_mem2 = h2_spike = torch.reshape(torch.zeros(batch_size, 11*11*32, device=device), (batch_size,-1,11,11))
        h3_mem1 = h3_mem2 = h3_spike = torch.reshape(torch.zeros(batch_size, 5*5*64, device=device), (batch_size,-1,5,5))
        h4_mem1 = h4_mem2 = h4_spike = torch.reshape(torch.zeros(batch_size, 2*2*10, device=device), (batch_size,-1,2,2))
        h5_sumspike = h5_mem1 = h5_mem2 = h5_spike = torch.reshape(torch.zeros(batch_size, 10, device=device), (batch_size,-1))
        # h1_sum = torch.zeros(1).cuda()
        # h2_sum = torch.zeros(1).cuda()
        # h3_sum = torch.zeros(1).cuda()
        # h4_sum = torch.zeros(1).cuda()
        # h5_sum = torch.zeros(1).cuda()

        for step in range(time_window): # simulation time steps

            x = input*max_firing_rate*delta_t > torch.rand(input.size(), device=device) # prob. firing

            h1_mem1, h1_mem2, h1_spike = mem_update(self.conv1, x.float(), h1_mem1, h1_mem2, h1_spike)
            h2_mem1, h2_mem2, h2_spike = mem_update(self.conv2, h1_spike, h2_mem1, h2_mem2, h2_spike)
            h3_mem1, h3_mem2, h3_spike = mem_update(self.conv3, h2_spike, h3_mem1, h3_mem2, h3_spike)
            h4_mem1, h4_mem2, h4_spike = mem_update(self.conv4, h3_spike, h4_mem1, h4_mem2, h4_spike)
            h5_mem1, h5_mem2, h5_spike = mem_update(self.fc1, torch.reshape(h4_spike,(batch_size,-1)), torch.reshape(h5_mem1,(batch_size,-1)), torch.reshape(h5_mem2,(batch_size,-1)), torch.reshape(h5_spike,(batch_size,-1)))
            h5_sumspike += h5_spike

            # h1_sum += torch.sum(h1_spike) / (h1_spike.numel() * time_window)
            # h2_sum += torch.sum(h2_spike) / (h2_spike.numel() * time_window)
            # h3_sum += torch.sum(h3_spike) / (h3_spike.numel() * time_window)
            # h4_sum += torch.sum(h4_spike) / (h4_spike.numel() * time_window)
            # h5_sum += torch.sum(h5_spike) / (h5_spike.numel() * time_window)

        # total_spike = h1_sum + h2_sum + h3_sum + h4_sum + h5_sum

        outputs = h5_sumspike / time_window

        return outputs

# os.environ['CUDA_VISIBLE_DEVICES'] = "3"
names = 'CNN_MNIST_cNet_archi_200hz_40ms_10ms_128step_0.1thr_adam_decay_e5'
data_path =  './raw/' #todo: input your data path
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

# pad = nn.ZeroPad2d(2)

train_dataset = torchvision.datasets.MNIST(root= data_path, train=True, download=True, transform=transforms.ToTensor())
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=0)
test_set = torchvision.datasets.MNIST(root= data_path, train=False, download=True, transform=transforms.ToTensor())
test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, shuffle=False, num_workers=0)

best_acc = 0  # best test accuracy
start_epoch = 0  # start from epoch 0 or last checkpoint epoch
acc_record = list([])
loss_train_record = list([])
loss_test_record = list([])

snn = SCNN()
snn.to(device)
criterion = nn.MSELoss()
# optimizer = torch.optim.Adam(snn.parameters(), lr=learning_rate)
optimizer = torch.optim.Adam(snn.parameters(), lr=learning_rate, weight_decay=1e-5)

# lamb = 1e-2
lamb = 0

for epoch in range(num_epochs):
    running_loss = 0
    start_time = time.time()
    for i, (images, labels) in enumerate(train_loader):
        snn.zero_grad()
        optimizer.zero_grad()

        images = images.float().to(device)
        outputs = snn(images)
        labels_ = torch.zeros(batch_size, 10).scatter_(1, labels.view(-1, 1), 1)
        loss = criterion(outputs.cpu(), labels_)
        running_loss += loss.item()
        loss.backward()
        optimizer.step()

        if (i+1)%100 == 0:
             print ('Epoch [%d/%d], Step [%d/%d], Loss: %.5f'
                    %(epoch+1, num_epochs, i+1, len(train_dataset)//batch_size,running_loss ))
             running_loss = 0
             print('Time elasped:', time.time()-start_time)
    correct = 0
    total = 0
    # optimizer = lr_scheduler(optimizer, epoch, learning_rate, 40)
    optimizer = lr_scheduler(optimizer, epoch, learning_rate, 15)

    with torch.no_grad():
        for batch_idx, (inputs, targets) in enumerate(test_loader):
            # inputs = pad(inputs)
            inputs = inputs.to(device)
            optimizer.zero_grad()
            outputs = snn(inputs)
            labels_ = torch.zeros(batch_size, 10).scatter_(1, targets.view(-1, 1), 1)
            loss = criterion(outputs.cpu(), labels_)
            _, predicted = outputs.cpu().max(1)
            total += float(targets.size(0))
            correct += float(predicted.eq(targets).sum().item())
            if batch_idx %100 ==0:
                acc = 100. * float(correct) / float(total)
                print(batch_idx, len(test_loader),' Acc: %.5f' % acc)

    print('Iters:', epoch,'\n\n\n')
    print('Test Accuracy of the model on the 10000 test images: %.3f' % (100 * correct / total))
    acc = 100. * float(correct) / float(total)
    acc_record.append(acc)

    if epoch % 5 == 0:
        print(acc)
        print('Saving..')
        state = {
            'net': snn.state_dict(),
            'acc': acc,
            'epoch': epoch,
            'acc_record': acc_record,
        }
        if not os.path.isdir('checkpoint'):
            os.mkdir('checkpoint')
        torch.save(state, './checkpoint/ckpt' + names + '.t7')
        best_acc = acc

        params = list(snn.parameters())
        w_0 = params[0].detach().cpu().flatten()
        w_1 = params[1].detach().cpu().flatten()
        w_2 = params[2].detach().cpu().flatten()
        w_3 = params[3].detach().cpu().flatten()
        w_4 = params[4].detach().cpu().flatten()
        plt.hist(w_0, bins=100)
        plt.show()
        plt.hist(w_1, bins=100)
        plt.show()
        plt.hist(w_2, bins=100)
        plt.show()
        plt.hist(w_3, bins=100)
        plt.show()
        plt.hist(w_4, bins=100)
        plt.show()


import paho.mqtt.client as mqtt #import the client1
from datetime import timezone
import time
import datetime

broker_address="localhost"
port = 1883
#broker_address="iot.eclipse.org"
print("creating new instance")
client = mqtt.Client("P1") #create new instance
print("connecting to broker")
client.connect(broker_address,port) #connect to broker
print("connected to:", broker_address)
while True:
    dt = datetime.datetime.now(timezone.utc)
    print(dt)
    time.sleep(1)
    print("Publishing message to topic", "utc_time")
    client.publish("utc_time", str(dt))


import math
import os
import torch
import torch.nn as nn
from collections import OrderedDict
import torch.utils.model_zoo as model_zoo
from torchvision.models.resnet import BasicBlock, Bottleneck, model_urls

def check_keys(model, pretrained_state_dict):
    ckpt_keys = set(pretrained_state_dict.keys())
    model_keys = set(model.state_dict().keys())
    used_pretrained_keys = model_keys & ckpt_keys
    unused_pretrained_keys = ckpt_keys - model_keys
    missing_keys = model_keys - ckpt_keys

    print('missing keys:{}'.format(missing_keys))

    print('=========================================')
    print('unused checkpoint keys:{}'.format(unused_pretrained_keys))
    # print('used keys:{}'.format(used_pretrained_keys))
    assert len(used_pretrained_keys) > 0, 'load NONE from pretrained checkpoint'
    return True

def load_pretrain(model, pretrained_dict):

    device = torch.cuda.current_device()

    check_keys(model, pretrained_dict)
    model.load_state_dict(pretrained_dict, strict=False)
    return model

class ResNet(nn.Module):
    """ ResNet network module. Allows extracting specific feature blocks."""
    def __init__(self, block, layers, output_layers, num_classes=1000, inplanes=64):
        self.inplanes = inplanes
        super(ResNet, self).__init__()
        self.output_layers = output_layers
        self.conv1 = nn.Conv2d(3, inplanes, kernel_size=7, stride=2, padding=3,
                               bias=False)
        self.bn1 = nn.BatchNorm2d(inplanes)
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
        self.layer1 = self._make_layer(block, inplanes, layers[0])
        self.layer2 = self._make_layer(block, inplanes*2, layers[1], stride=2)
        self.layer3 = self._make_layer(block, inplanes*4, layers[2], stride=2)
        self.layer4 = self._make_layer(block, inplanes*8, layers[3], stride=2)
        # self.avgpool = nn.AvgPool2d(7, stride=1)
        self.avgpool = nn.AdaptiveAvgPool2d((1,1))
        self.fc = nn.Linear(inplanes*8 * block.expansion, num_classes)

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()

    def _make_layer(self, block, planes, blocks, stride=1):
        downsample = None
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                nn.Conv2d(self.inplanes, planes * block.expansion,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(planes * block.expansion),
            )

        layers = []
        layers.append(block(self.inplanes, planes, stride, downsample))
        self.inplanes = planes * block.expansion
        for i in range(1, blocks):
            layers.append(block(self.inplanes, planes))

        return nn.Sequential(*layers)

    def _add_output_and_check(self, name, x, outputs, output_layers):
        if name in output_layers:
            outputs[name] = x
        return len(output_layers) == len(outputs)

    def forward(self, x, output_layers=None):
        """ Forward pass with input x. The output_layers specify the feature blocks which must be returned """
        outputs = OrderedDict()

        if output_layers is None:
            output_layers = self.output_layers

        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)

        if self._add_output_and_check('conv1', x, outputs, output_layers):
            return outputs

        x = self.maxpool(x)

        x = self.layer1(x)

        if self._add_output_and_check('layer1', x, outputs, output_layers):
            return outputs

        x = self.layer2(x)

        if self._add_output_and_check('layer2', x, outputs, output_layers):
            return outputs

        x = self.layer3(x)

        if self._add_output_and_check('layer3', x, outputs, output_layers):
            return outputs

        x = self.layer4(x)

        if self._add_output_and_check('layer4', x, outputs, output_layers):
            return outputs

        x = self.avgpool(x)
        x = x.view(x.size(0), -1)
        x = self.fc(x)

        if self._add_output_and_check('fc', x, outputs, output_layers):
            return outputs

        if len(output_layers) == 1 and output_layers[0] == 'default':
            return x

        raise ValueError('output_layer is wrong.')

def resnet18(output_layers=None, pretrained=False):
    """Constructs a ResNet-18 model.
    """

    if output_layers is None:
        output_layers = ['default']
    else:
        for l in output_layers:
            if l not in ['conv1', 'layer1', 'layer2', 'layer3', 'layer4', 'fc']:
                raise ValueError('Unknown layer: {}'.format(l))

    model = ResNet(BasicBlock, [2, 2, 2, 2], output_layers)

    if pretrained:
        model.load_state_dict(model_zoo.load_url(model_urls['resnet18']))
    return model

def resnet50(output_layers=None, pretrained_path='pretrain/ResNet50.pth', ):
    """Constructs a ResNet-50 model.
    """

    if output_layers is None:
        output_layers = ['default']
    else:
        for l in output_layers:
            if l not in ['conv1', 'layer1', 'layer2', 'layer3', 'layer4', 'fc']:
                raise ValueError('Unknown layer: {}'.format(l))

    model = ResNet(Bottleneck, [3, 4, 6, 3], output_layers)
    if pretrained_path and os.path.exists(pretrained_path):
        device = torch.cuda.current_device()
        pretrained_dict = torch.load(pretrained_path, map_location=lambda storage, loc: storage.cuda(device))
        # pretrained_dict.update({'conv1_weight':new_conv1_weight})
        # pretrained_dict['conv1_weight'] = new_conv1_weight
        new_dict = OrderedDict()
        # for key in pretrained_dict.keys():
        #     if key == 'conv1.weight':
        #         conv1_weight = pretrained_dict[key].contiguous()
        #         new_conv1_weight = torch.cat((conv1_weight, conv1_weight), dim=1)
        #         new_dict[key] = new_conv1_weight
        #     else:
        #         new_dict[key] = pretrained_dict[key]
        for key in pretrained_dict['net'].keys():
            if key.startswith('feature_extractor'):
                new_dict[key.split('feature_extractor.')[-1]] = pretrained_dict['net'][key]
            else:
                new_dict[key] = pretrained_dict['net'][key]

        # for key in new_dict.keys():
        # print("key: {} shape: {}".format(key, new_dict[key].shape))
        load_pretrain(model, new_dict)
    else:
        print('attention: no imagenet pretrain, testing?')
    return model


import intake
import param

from ..util import get_dataframe_schema
from .base import Source, cached

class IntakeBaseSource(Source):

    load_schema = param.Boolean(default=True, doc="""
        Whether to load the schema""")

    __abstract = True

    def _read(self, table, dask=True):
        try:
            entry = self.cat[table]
        except KeyError:
            raise KeyError(f"'{table}' table could not be found in Intake "
                           "catalog. Available tables include: "
                           f"{list(self.cat)}.")
        if self.dask or dask:
            try:
                return entry.to_dask()
            except Exception:
                if self.dask:
                    self.param.warning(f"Could not load {table} table with dask.")
                pass
        return entry.read()

    def get_tables(self):
        return list(self.cat)

    def get_schema(self, table=None):
        schemas = {}
        for entry in list(self.cat):
            if table is not None and entry != table:
                continue
            data = self.get(entry, __dask=True)
            if self.load_schema:
                schemas[entry] = {}
            else:
                schemas[entry] = get_dataframe_schema(data)['items']['properties']
        return schemas if table is None else schemas[table]

    @cached(with_query=False)
    def get(self, table, **query):
        dask = query.pop('__dask', self.dask)
        df = self._read(table)
        return df if dask or not hasattr(df, 'compute') else df.compute()

class IntakeSource(IntakeBaseSource):
    """
    An IntakeSource loads data from an Intake catalog.
    """

    catalog = param.Dict(doc="An inlined Catalog specification.")

    dask = param.Boolean(default=False, doc="""
        Whether to return a dask DataFrame.""")

    uri = param.String(doc="URI of the catalog file.")

    source_type = 'intake'

    def __init__(self, **params):
        super().__init__(**params)
        if self.uri and self.catalog:
            raise ValueError("Either specify a Catalog uri or an "
                             "inlined catalog, not both.")
        elif self.uri:
            self.cat = intake.open_catalog(self.uri)
        elif self.catalog:
            context = {'root': self.root}
            result = intake.catalog.local.CatalogParser(self.catalog, context=context)
            if result.errors:
                raise intake.catalog.exceptions.ValidationError(
                    "Catalog '{}' has validation errors:\n\n{}"
                    "".format(self.catalog, "\n".join(result.errors)), result.errors)
            cfg = result.data
            del cfg['plugin_sources']
            entries = {entry.name: entry for entry in cfg.pop('data_sources')}
            self.cat = intake.catalog.Catalog(**cfg)
            self.cat._entries = entries

class IntakeDremioSource(IntakeBaseSource):

    cert = param.String(default="Path to certificate file")

    dask = param.Boolean(default=False, doc="""
        Whether to return a dask DataFrame.""")

    uri = param.String(doc="URI of the catalog file.")

    tls = param.Boolean(default=False, doc="Enable encryption")

    source_type = 'intake_dremio'

    def __init__(self, **params):
        from intake_dremio.dremio_cat import DremioCatalog
        super().__init__(**params)
        self.cat = DremioCatalog(self.uri, cert=self.cert, tls=self.tls)


import os
import pydot
import subprocess

# out = subprocess.Popen(['wc', '-l', 'TripCombo.txt'],
#                        stdout=subprocess.PIPE,
#                        stderr=subprocess.STDOUT)
#
# stdout, stderr = out.communicate()
# print(stdout)

os.environ["PATH"] += os.pathsep + 'C:/Program Files (x86)/Graphviz2.38/bin/'

def returnParentheticalFormat(fileName):
    parentheticalFormat = ''
    for line in reversed(list(open(fileName))):
        if line.__contains__('eNewick'):
            parentheticalFormat = line[28:]
            # print(parentheticalFormat)
            # print(line.rstrip())
    return parentheticalFormat.rstrip()

def convertDotToPNG(fileName):
    (graph,) = pydot.graph_from_dot_file(fileName)
    graph.write_png('cExample1.png')

def convertDotToPNGJulia(fileName):
    paren = returnParentheticalFormat(fileName)
    with open("NetworkParen.net", "w") as text_file:
        text_file.write(paren)
    # write it to a file
    # then use the julia command
    cmd = 'julia plot-network.jl NetworkParen.net'
    os.system(cmd)

def removoeLeaves(leafNodes):
    with open("leaves.txt", "w") as text_file:
        text_file.write(leafNodes)
    # write it to a file
    # then use the julia command
    cmd = 'julia .\\remove-leaves.jl .\\NetworkParen.net .\\leaves.txt'
    os.system(cmd)
    # convert Parenthetical format to png
    cmd = 'julia plot-network.jl reduced-net.txt'
    os.system(cmd)

def tripletsToDot(tripletsFName):
    cmd = 'java -jar Lev1athan.jar ' + tripletsFName + ' > cExample1.dot'
    os.system(cmd)

if __name__ == '__main__':
    print("Hello")
    tripletsToDot('cExample1.trips')
    # convertDotToPNG('cExample1.dot')
    convertDotToPNGJulia('cExample1.dot')
    # removoeLeaves('1\n3\n4')
    print("Hello")

#  1) Return Parenthetical fiel and return the image
# Download image or text file
# Two buttons one for image other for text
#  Filtering the graph
# Remove Leaf Nodes
# Change Root and checbox to filter leaves

# checkbox with all the leaves and uncheck the nodes

# last line if you use please site the following
# manuscript
# Download button to


import cv2
import os
import argparse
import pandas as pd
import numpy as np

from merge_csv import merge_csv

CUR_DIR = os.path.dirname(os.path.abspath(__file__))

# Create argument parser and parse args
ap = argparse.ArgumentParser()
ap.add_argument("-d", "--data", required=False,
	help="path to csv metadata file")
args = vars(ap.parse_args())

if not args['data']:
    merge_csv()
    args['data'] = os.path.join(CUR_DIR, 'data', 'data.csv')

data = pd.read_csv(args['data'])
for idx, row in data.iterrows():
    filename = os.path.join(CUR_DIR, 'data', row['image'])
    face = tuple(row[1:5])
    landmarks = np.array(row[5:141]).reshape((68, 2))

    # Draw image with detected landmarks
    image = cv2.imread(filename)
    (x, y, w, h) = face
    cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)
    for (x, y) in landmarks:
        cv2.circle(image, (x, y), 1, (0, 0, 255), -1)
    cv2.imshow('Landmarks', image)
    if cv2.waitKey() == 27:
        break