import ctypes
import threading
import time
from contextlib import contextmanager
from queue import Queue
from typing import Any, Callable, Dict, Generator, Iterable, List, Optional, Tuple, Union, cast

import attr
import hypothesis
import hypothesis.errors
import requests
from requests.auth import AuthBase

from .._hypothesis import make_test_or_exception
from ..constants import USER_AGENT
from ..exceptions import InvalidSchema
from ..loaders import from_uri
from ..models import Case, Endpoint, Status, TestResult, TestResultSet
from ..schemas import BaseSchema
from ..utils import capture_hypothesis_output, get_base_url
from . import events
from .checks import DEFAULT_CHECKS

DEFAULT_DEADLINE = 500  # pragma: no mutate

Auth = Union[Tuple[str, str], AuthBase]  # pragma: no mutate

def get_hypothesis_settings(hypothesis_options: Optional[Dict[str, Any]] = None) -> hypothesis.settings:
    # Default settings, used as a parent settings object below
    settings = hypothesis.settings(deadline=DEFAULT_DEADLINE)
    if hypothesis_options is not None:
        settings = hypothesis.settings(settings, **hypothesis_options)
    return settings

@attr.s
class BaseRunner:
    schema: BaseSchema = attr.ib()
    checks: Iterable[Callable] = attr.ib()
    hypothesis_settings: hypothesis.settings = attr.ib(converter=get_hypothesis_settings)
    auth: Optional[Auth] = attr.ib(default=None)
    headers: Optional[Dict[str, Any]] = attr.ib(default=None)
    request_timeout: Optional[int] = attr.ib(default=None)
    seed: Optional[int] = attr.ib(default=None)

    def execute(self,) -> Generator[events.ExecutionEvent, None, None]:
        """Common logic for all runners."""
        results = TestResultSet()

        initialized = events.Initialized(
            results=results, schema=self.schema, checks=self.checks, hypothesis_settings=self.hypothesis_settings
        )
        yield initialized

        yield from self._execute(results)

        yield events.Finished(results=results, schema=self.schema, running_time=time.time() - initialized.start_time)

    def _execute(self, results: TestResultSet) -> Generator[events.ExecutionEvent, None, None]:
        raise NotImplementedError

@attr.s(slots=True)
class SingleThreadRunner(BaseRunner):
    """Fast runner that runs tests sequentially in the main thread."""

    def _execute(self, results: TestResultSet) -> Generator[events.ExecutionEvent, None, None]:
        with get_session(self.auth, self.headers) as session:
            for endpoint, test in self.schema.get_all_tests(single_test, self.hypothesis_settings, self.seed):
                for event in run_test(self.schema, endpoint, test, self.checks, session, results, self.request_timeout):
                    yield event
                    if isinstance(event, events.Interrupted):
                        return

def thread_task(
    tasks_queue: Queue,
    events_queue: Queue,
    schema: BaseSchema,
    checks: Iterable[Callable],
    settings: hypothesis.settings,
    auth: Optional[Auth],
    headers: Optional[Dict[str, Any]],
    request_timeout: Optional[int],
    seed: Optional[int],
    results: TestResultSet,
) -> None:
    """A single task, that threads do.

    Pretty similar to the default one-thread flow, but includes communication with the main thread via the events queue.
    """
    # pylint: disable=too-many-arguments
    with get_session(auth, headers) as session:
        with capture_hypothesis_output():
            while not tasks_queue.empty():
                endpoint = tasks_queue.get()
                test = make_test_or_exception(endpoint, single_test, settings, seed)
                for event in run_test(schema, endpoint, test, checks, session, results, request_timeout):
                    events_queue.put(event)

def stop_worker(thread_id: int) -> None:
    """Raise an error in a thread so it is possible to asynchronously stop thread execution."""
    ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(thread_id), ctypes.py_object(SystemExit))

class ThreadInterrupted(Exception):
    """Special exception when worker thread received SIGINT."""

@attr.s(slots=True)
class ThreadPoolRunner(BaseRunner):
    """Spread different tests among multiple worker threads."""

    workers_num: int = attr.ib(default=2)

    def _execute(self, results: TestResultSet) -> Generator[events.ExecutionEvent, None, None]:
        """All events come from a queue where different workers push their events."""
        tasks_queue = self._get_tasks_queue()
        # Events are pushed by workers via a separate queue
        events_queue: Queue = Queue()
        workers = self._init_workers(tasks_queue, events_queue, results)

        def stop_workers() -> None:
            for worker in workers:
                # workers are initialized at this point and `worker.ident` is set with an integer value
                ident = cast(int, worker.ident)
                stop_worker(ident)
                worker.join()

        is_finished = False
        try:
            while not is_finished:
                # Sleep is needed for performance reasons
                # each call to `is_alive` of an alive worker waits for a lock
                # iterations without waiting are too frequent and a lot of time will be spent on waiting for this locks
                time.sleep(0.001)
                is_finished = all(not worker.is_alive() for worker in workers)
                while not events_queue.empty():
                    event = events_queue.get()
                    yield event
                    if isinstance(event, events.Interrupted):
                        # Thread received SIGINT
                        # We could still have events in the queue, but ignore them to keep the logic simple
                        # for now, could be improved in the future to show more info in such corner cases
                        raise ThreadInterrupted
        except ThreadInterrupted:
            stop_workers()
        except KeyboardInterrupt:
            stop_workers()
            yield events.Interrupted(results=results, schema=self.schema)

    def _get_tasks_queue(self) -> Queue:
        """All endpoints are distributed among all workers via a queue."""
        tasks_queue: Queue = Queue()
        tasks_queue.queue.extend(self.schema.get_all_endpoints())
        return tasks_queue

    def _init_workers(self, tasks_queue: Queue, events_queue: Queue, results: TestResultSet) -> List[threading.Thread]:
        """Initialize & start workers that will execute tests."""
        workers = [
            threading.Thread(
                target=thread_task,
                kwargs={
                    "tasks_queue": tasks_queue,
                    "events_queue": events_queue,
                    "schema": self.schema,
                    "checks": self.checks,
                    "settings": self.hypothesis_settings,
                    "auth": self.auth,
                    "headers": self.headers,
                    "request_timeout": self.request_timeout,
                    "seed": self.seed,
                    "results": results,
                },
            )
            for _ in range(self.workers_num)
        ]
        for worker in workers:
            worker.start()
        return workers

def execute_from_schema(
    schema: BaseSchema,
    checks: Iterable[Callable],
    *,
    workers_num: int = 1,
    hypothesis_options: Optional[Dict[str, Any]] = None,
    auth: Optional[Auth] = None,
    headers: Optional[Dict[str, Any]] = None,
    request_timeout: Optional[int] = None,
    seed: Optional[int] = None,
) -> Generator[events.ExecutionEvent, None, None]:
    """Execute tests for the given schema.

    Provides the main testing loop and preparation step.
    """
    runner: BaseRunner
    if workers_num > 1:
        runner = ThreadPoolRunner(
            schema, checks, hypothesis_options, auth, headers, request_timeout, seed, workers_num=workers_num
        )
    else:
        runner = SingleThreadRunner(schema, checks, hypothesis_options, auth, headers, request_timeout, seed)

    yield from runner.execute()

def run_test(
    schema: BaseSchema,
    endpoint: Endpoint,
    test: Union[Callable, InvalidSchema],
    checks: Iterable[Callable],
    session: requests.Session,
    results: TestResultSet,
    request_timeout: Optional[int],
) -> Generator[events.ExecutionEvent, None, None]:
    """A single test run with all error handling needed."""
    # pylint: disable=too-many-arguments
    result = TestResult(endpoint=endpoint, schema=schema)
    yield events.BeforeExecution(results=results, schema=schema, endpoint=endpoint)
    hypothesis_output: List[str] = []
    try:
        if isinstance(test, InvalidSchema):
            status = Status.error
            result.add_error(test)
        else:
            with capture_hypothesis_output() as hypothesis_output:
                test(session, checks, result, request_timeout)
            status = Status.success
    except AssertionError:
        status = Status.failure
    except hypothesis.errors.Flaky:
        status = Status.error
        result.mark_errored()
        # Sometimes Hypothesis detects inconsistent test results and checks are not available
        if result.checks:
            flaky_example = result.checks[-1].example
        else:
            flaky_example = None
        result.add_error(
            hypothesis.errors.Flaky(
                "Tests on this endpoint produce unreliable results: \n"
                "Falsified on the first call but did not on a subsequent one"
            ),
            flaky_example,
        )
    except hypothesis.errors.Unsatisfiable:
        # We need more clear error message here
        status = Status.error
        result.add_error(hypothesis.errors.Unsatisfiable("Unable to satisfy schema parameters for this endpoint"))
    except KeyboardInterrupt:
        yield events.Interrupted(results=results, schema=schema)
        return
    except Exception as error:
        status = Status.error
        result.add_error(error)
    # Fetch seed value, hypothesis generates it during test execution
    result.seed = getattr(test, "_hypothesis_internal_use_seed", None) or getattr(
        test, "_hypothesis_internal_use_generated_seed", None
    )
    results.append(result)
    yield events.AfterExecution(
        results=results, schema=schema, endpoint=endpoint, status=status, hypothesis_output=hypothesis_output
    )

def execute(  # pylint: disable=too-many-arguments
    schema_uri: str,
    checks: Iterable[Callable] = DEFAULT_CHECKS,
    api_options: Optional[Dict[str, Any]] = None,
    loader_options: Optional[Dict[str, Any]] = None,
    hypothesis_options: Optional[Dict[str, Any]] = None,
    loader: Callable = from_uri,
) -> TestResultSet:
    generator = prepare(
        schema_uri=schema_uri,
        checks=checks,
        api_options=api_options,
        loader_options=loader_options,
        hypothesis_options=hypothesis_options,
        loader=loader,
    )
    all_events = list(generator)
    finished = all_events[-1]
    return finished.results

def prepare(  # pylint: disable=too-many-arguments
    schema_uri: str,
    checks: Iterable[Callable] = DEFAULT_CHECKS,
    workers_num: int = 1,
    api_options: Optional[Dict[str, Any]] = None,
    loader_options: Optional[Dict[str, Any]] = None,
    hypothesis_options: Optional[Dict[str, Any]] = None,
    loader: Callable = from_uri,
    seed: Optional[int] = None,
) -> Generator[events.ExecutionEvent, None, None]:
    """Prepare a generator that will run test cases against the given API definition."""
    api_options = api_options or {}
    loader_options = loader_options or {}

    if "base_url" not in loader_options:
        loader_options["base_url"] = get_base_url(schema_uri)
    schema = loader(schema_uri, **loader_options)
    return execute_from_schema(
        schema, checks, hypothesis_options=hypothesis_options, seed=seed, workers_num=workers_num, **api_options
    )

def single_test(
    case: Case,
    session: requests.Session,
    checks: Iterable[Callable],
    result: TestResult,
    request_timeout: Optional[int],
) -> None:
    """A single test body that will be executed against the target."""
    # pylint: disable=too-many-arguments
    timeout = prepare_timeout(request_timeout)
    response = case.call(session=session, timeout=timeout)
    errors = None

    for check in checks:
        check_name = check.__name__
        try:
            check(response, result)
            result.add_success(check_name, case)
        except AssertionError as exc:
            errors = True  # pragma: no mutate
            result.add_failure(check_name, case, str(exc))

    if errors is not None:
        # An exception needed to trigger Hypothesis shrinking & flaky tests detection logic
        # The message doesn't matter
        raise AssertionError

def prepare_timeout(timeout: Optional[int]) -> Optional[float]:
    """Request timeout is in milliseconds, but `requests` uses seconds"""
    output: Optional[Union[int, float]] = timeout
    if timeout is not None:
        output = timeout / 1000
    return output

@contextmanager
def get_session(
    auth: Optional[Auth] = None, headers: Optional[Dict[str, Any]] = None
) -> Generator[requests.Session, None, None]:
    with requests.Session() as session:
        if auth is not None:
            session.auth = auth
        session.headers["User-agent"] = USER_AGENT
        if headers is not None:
            session.headers.update(**headers)
        yield session


import numpy as np
import matplotlib.pyplot as plt

# Here we simulate just the motor

# The goal is to turn the motor to a specified position
# Input:            motor torque
# State variables:  shaft angular position, shaft angular velocity
# Output:           shaft position

# Plan:
# 1. Numerically simulate the system
# 2. Add a controller
# 3. Simulate the system with the controller
# -----------------------------------------------------------------------------

# 1. Numerically simulate the system
# Differential equation for system:
# T = mmI*d^2(theta)/dt^2
# T = torque
# mmI = mass moment of inertia
# theta = angular position (radians)

# Writing as two first order ODEs
# T = mmI*d(x2)/dt
# x2 = d(x1)/dt
# where x1 = theta

# Approximating derivatives using Euler's methods, and solving for x1 (see Maple file)
# x2 = np.append(x2,(T[i]*delT+mmI*x2[i-1])/mmI) # angular velocity
# x1 = np.append(x1,delT*x2[i]+x1[i-1])

# Simulate the response of the motor to one torque input
def simOneInteraction(x1, x2, T, mmI, delT):
    i = len(x1)
    x2 = np.append(x2,(T*delT+mmI*x2[i-1])/mmI) # angular velocity
    x1 = np.append(x1,delT*x2[i]+x1[i-1])
    return [x1, x2]

# Plot results of motor simulation
def plotSim(x1,x2,Tseq,numIter,delT):
    t = np.arange(0, delT*numIter, delT);
    theta = x1
    plt.plot(t,theta,'b',label = 'Angular Position')
    plt.plot(t,x2,'r',label = 'Angular Velocity')
    plt.plot(t,Tseq,'g',label = 'Torque')
    plt.legend(loc = 4)
    plt.title('Motor Shaft Simulation')
    plt.xlabel('Time (s)')
    plt.ylabel('Angular position (rad), Angular velocity (rad/s)')
    plt.show()

# Demo usage
def demoSim():
    # System parameters
    mmI = 1 # kg*m^2

    # Length and precision of simulation
    numIter = 2000
    delT = 0.1 # Time interval between steps, in sconds

    # Simulate
    x1 = np.array([0]); # Initial position
    x2 = np.array([0]); # Initial velocity
    xr = 10; # Desired position

    Tseq = np.array([0]);
    for i in range(numIter-1):
        # Control policy (proportional control)
        kp = 0.01
        eNow = xr - x1[i]
        if (i > 0): # PD control
            ePrev = xr - x1[i-1]
            kd = 1
            T = kp*eNow + kd*(eNow - ePrev )
        else: # P control
            T = kp*eNow

        # Keep track of torque used
        Tseq = np.append(Tseq,T)
        (x1,x2) = simOneInteraction(x1, x2, T, mmI, delT)

    plotSim(x1,x2,Tseq,numIter,delT)

demoSim()


import time
import numpy as np
import networkx as nx
from networkx.algorithms.flow import maximum_flow
from evacuation import FlowGraph, max_flow

def transform_matrix_to_adj(matrix, num_of_vertices, m, limit):
    if m > limit:
        edges = set(np.random.permutation(range(m))[:limit])
    adjacency = [[] for i in range(num_of_vertices)]
    costs = [[] for i in range(num_of_vertices)]
    for i in range(0, matrix.shape[0]):
        for j in range(0, matrix.shape[1]):
            if matrix[i, j] and ((m > limit and i*matrix.shape[0]+j in edges) or m <= limit):
                adjacency[i].append(j)
                costs[i].append(matrix[i, j])
    return adjacency, costs

def build_graph(num_of_vertices, adjacency, costs):
    graph = FlowGraph(num_of_vertices)
    edges = [(i, m, w) for i, n in enumerate(zip(adjacency, costs)) for m, w in zip(*n)]
    for (i, m, w) in edges:
        graph.add_edge(i, m, w)
    return graph

def nx_flow(num_of_vertices, adjacency, costs):
    G = nx.DiGraph()
    G.add_nodes_from(list(range(0, num_of_vertices)))
    edges = [(i, m, w) for i, n in enumerate(zip(adjacency, costs)) for m, w in zip(*n)]
    G.add_weighted_edges_from(edges)
    flow, _ = maximum_flow(G, 0, num_of_vertices-1, capacity='weight')
    return flow

if __name__ == '__main__':
    for ri in range(10**6):
        np.random.seed(ri)
        while True:
            s0 = time.time()
            n = np.random.randint(2, 10 ** 2, 1)[0]
            matrix = np.random.randint(0, 2, (n, n))
            m = np.sum(matrix)
            matrix_costs = np.random.randint(0, 1000, (n, n))
            matrix = np.multiply(matrix, matrix_costs)
            adjacency, costs = transform_matrix_to_adj(matrix, n, m, 10 ** 4)

            graph = build_graph(n, adjacency, costs)
            s0 = time.time()
            my_result, iters = max_flow(graph, 0, graph.size() - 1)
            s1 = time.time()
            result = nx_flow(n, adjacency, costs)
            s2 = time.time()

            print(ri, s1-s0, s2-s1, n, m, result, my_result, iters)

            assert my_result == result


import requests
import zipfile
import io

link_list = [
    "http://vap.aau.dk/wp-content/uploads/VAPRBGD/(2012-05-18)(151650).zip"]

print(len(link_list))

for link in link_list:
    print("downloading....{}".format(link))
    r = requests.get(link, stream=True)
    z = zipfile.ZipFile(io.BytesIO(r.content))
    z.extractall("faceid_train")
    print("extracted....{}".format(link))

# for link in val_list:
#     print("downloading....{}".format(link))
#     r = requests.get(link, stream=True)
#     z = zipfile.ZipFile(io.BytesIO(r.content))
#     z.extractall("faceid_val")
#     print("extracted....{}".format(link))


import json
import os
from flask import Flask, render_template, request, redirect, Response, jsonify, send_from_directory, flash, url_for
import pandas as pd
import numpy as np
from natsort import natsorted
from getRequestedData import getRequestedData
import datetime
from werkzeug.utils import secure_filename
from collections import defaultdict
import logging
logging.basicConfig(filename='error.log', level=logging.DEBUG)
# from getPredictions import getPredictions

app = Flask(__name__)

# Function to reverse a dummy (one-hot encoded) dataframe
def reverse_dummy(df_dummies):
    pos = defaultdict(list)
    vals = defaultdict(list)

    for i, c in enumerate(df_dummies.columns):
        if "_" in c:
            k, v = c.split("_", 1)
            pos[k].append(i)
            vals[k].append(v)
        else:
            pos["_"].append(i)

    df = pd.DataFrame({k: pd.Categorical.from_codes(
                              np.argmax(df_dummies.iloc[:, pos[k]].values, axis=1),
                              vals[k])
                      for k in vals})

    df[df_dummies.columns[pos["_"]]] = df_dummies.iloc[:, pos["_"]]
    return df

# Read_data_again changes the values of the global variables for the newly uploaded file. After this, the tool will display data of the uploaded file
# The code is same as the main function, we need to do it again so as to read the newly uploaded file
def read_data_again():
    global data
    global columns
    global data_dummy
    global column_dummy
    global on_cols
    global off_cols
    global data_tosend
    global dataframes
    global blacklist_cols
    global history_global

    # Sampling data and read in chunks in case of a large file
    chunks = pd.read_csv('dataset/uploaded_file.csv', chunksize=100000)
    # Initialize an empty dataframe with columns from the file
    data = pd.DataFrame(columns=list(chunks.get_chunk(0)))
    # Iterate over the chunks and append every 10k rows sampled form a 100k chunk 10 times max
    for i,chunk in enumerate(chunks):
        if i==10:
            break # We have a dataframe with 100k rows already
        if chunk.shape[0] >= 10000:
            data = data.append(chunk.sample(10000))
        else:
            data = data.append(chunk)
            break
    # print(data.shape)
    # Removing special characters from column names
    for col in data.columns:
        str = ''.join(e for e in col if e.isalnum())
        data.rename(columns={col:str}, inplace=True)
    columns = list(data.columns) # Crete a list of columns in the dataset
    temp_data = data
    data = data[columns[:-1]]
    # List numerical and categorical variables
    num_cols = list(data._get_numeric_data().columns)
    cat_cols = list(set(data.columns) - set(num_cols))
    # Converting the numerical variables to categorical
    data = data.dropna()
    for col in num_cols:
        data[col] = pd.cut(data[col], 5, labels=['low', 'midlow', 'mid', 'midhigh', 'high'])
    # Sort columns based on number of categories
    num_cat = {}
    for col in list(data.columns):
        if len(data[col].value_counts()) in num_cat.keys():
            num_cat[len(data[col].value_counts())].append(col)
        else:
            num_cat[len(data[col].value_counts())] = [col]
    # Reorder the dataframe based on ranked variables
    cols = []
    for key in sorted(num_cat.keys()):
        cols += num_cat[key]
    data = data[cols]
    data['Throughput'] = temp_data[columns[-1]]
    # for col in columns[:-1]: # For all columns except throughput
    #     print(col)
    #     data[col] = data[col].apply(str) # Change the datatype for each column to be of type string so that there are no conflicts when performing calculations on each of the columns
    data_dummy = pd.get_dummies(data) #Since all the data in categorical, this creates a boolean dummy dataframe by creating columns of all the categories for each variable. This creates a column for each bar displayed
    data_dummy = data_dummy.loc[:, (data_dummy != 0).any(axis=0)]
    column_dummy = list(data_dummy.columns)
    if len(column_dummy) > 50:
        column_dummy = column_dummy[:50]
    data_dummy = data_dummy[column_dummy]
    data = reverse_dummy(data_dummy)
    on_cols = [] # List of bars that are turned on. Initially, all the bars are on. We start from 1st index because column_dummy[0] = Throughput
    off_cols = [] # No bars are turned off initially
    columns = list(data.columns)
    data_tosend = {'columns': columns[:-1]}
    data_tosend['Max Cols'] = len(column_dummy)-1 # Total number of bars to display i.e. all columns in the dummy dataframe except throughput
    dataframes = {} # This variable stores all variables in the dataset and the statistics of throughput for each of these variables. It is a dict of dataframes
    blacklist_cols = [] # It stores the variables for which the buttons are turned 'off'.
    history_global = [] # List used to store on_cols, off_cols and blacklist_cols for each state. Used for blockchain plot

UPLOAD_FOLDER = os.path.join(app.root_path, "dataset") # Path to the folder where the uploaded dataste would be saved
ALLOWED_EXTENSIONS = ['csv'] # Extensions allowed for the datasets to be uploaded to ICE server
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
# function to check whether the uploaded dataset is valid
def allowed_file(filename):
    return '.' in filename and \
    filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

# Function to handle uploaded file
@app.route('/upload', methods=['POST'])
def upload():
    if request.method == 'POST':
        # check if the post request has the file part
        if 'file' not in request.files:
            flash('No file part')
            return redirect(request.url)
        file = request.files['file']
        # if user does not select file, browser also
        # submit an empty part without filename
        if file.filename == '':
            flash('No selected file')
            return redirect(request.url)
        if file and allowed_file(file.filename):
            filename = secure_filename(file.filename)
            # The file is saved with the name "uploaded_file.csv" inside the dataset folder
            file.save(os.path.join(app.config['UPLOAD_FOLDER'], 'uploaded_file.csv'))
            read_data_again()
            return redirect(url_for('index'))

# @app.route('/uploads/<filename>')
# def uploaded_file(filename):
#     return send_from_directory(app.config['UPLOAD_FOLDER'],
#                                filename)

@app.route("/blockchain", methods = ['POST']) # This method is called when the user clicks on a blockchain element
def getBlockchainData():
    # Variables that we need to recalculate the dataset are defined as global
    global history_global
    global data_tosend
    global data_dummy
    global dataframes
    global columns

    # Filtered Data to be calculated based on "on_cols, off_cols and blacklist_cols"
    filtered_data = data_dummy
    # Initially we assume that the configuration user selected exists.
    data_tosend['No Config Exist'] = 'False'
    # Get the index of the circle user clicked on the blockchain
    history_index = request.form['index']
    # Get details of "on_cols, off_cols and blacklist_cols" from the history array using the index obtained
    on_cols = history_global[int(history_index)]['on_cols']
    off_cols = history_global[int(history_index)]['off_cols']
    blacklist_cols = history_global[int(history_index)]['blacklist_cols']
    # Recalculate the data at that stage
    data_tosend = getRequestedData(on_cols, off_cols, blacklist_cols, filtered_data, data_tosend, '', '', '', dataframes, history_global, columns)
    return jsonify(data_tosend)

@app.route("/", methods = ['POST', 'GET']) # View function for the app
def index():
    global on_cols
    global off_cols
    global history_global
    global data_tosend
    data_tosend['No Config Exist'] = 'False' # By default, we assume that a configuration selected by the user exists. If it doesn't, we change it later in the code
    if request.method == 'POST': # Collect the Post request when the user clicks a bar
        column_received = request.form['column'] # Get the variable selected by the user
        value_received = request.form['value'] # Get the category within the variable selected by the user
        switch_received = request.form['switch'] # "on" or "off"
        logging.info(column_received)
        logging.info(value_received)
        logging.info(switch_received)
        now = datetime.datetime.now() # Get current date and time from the user to log in the file for user study
        logging.info(str(now.hour)+':'+str(now.minute)+':'+str(now.second))
        filtered_data = data_dummy # Create a new dataframe to edit the values
        if value_received == 'all':
            unique_values = list(data[column_received].unique()) # Create a list of categories in the selected variable
            '''
            Notice that in this case, turning on/off a variable completely doesn't change the results. Turning on all features mean that we are adding that variable in the dataset.
            On the other hand, turning off a variable means that we don't count that variable in our calculations i.e. it is wildcarded. In this case, the variable remain in the dataset but is
            hidden from calculations. So, in both cases, the variable remains in the dataset.
            '''
            if switch_received == 'off':
                if column_received not in blacklist_cols:
                    blacklist_cols.append(column_received)
                for value in unique_values:
                    try:
                        on_cols.remove(column_received+'_'+value) # Remove all the categories from on_cols
                    except ValueError:
                        pass
                    if column_received+'_'+value not in off_cols: # Add all the categories to off_cols if not already present
                        off_cols.append(column_received+'_'+value)

            else:
                try:
                    blacklist_cols.remove(column_received)
                except ValueError:
                    pass
                for value in unique_values:
                    try:
                        off_cols.remove(column_received+'_'+value) # Remove the categories from off_cols
                    except ValueError:
                        pass
                    if column_received+'_'+value not in on_cols: # Add the categories to on_cols if not already present
                        on_cols.append(column_received+'_'+value)
        else:
            if switch_received == 'off': # Remove the selected bar from on_cols and add to off_cols list
                try:
                    on_cols.remove(column_received+'_'+value_received)
                except ValueError:
                    pass
                off_cols.append(column_received+'_'+value_received)
            else: # Add the bar to on_cols and remove from off_cols
                '''
                It is important to remove the blacklisted variable at this stage because if the user clicked a button for a variable to turn them off, then he again clicks on a single bar
                for the same variable. In this case, the variable is back to scene and it is important to remove this column from blacklisted variables for proper
                functioning of the app.
                '''
                if column_received in blacklist_cols: # Remove the variable if it is blacklisted
                    blacklist_cols.remove(column_received)
                on_cols.append(column_received+'_'+value_received)
                off_cols.remove(column_received+'_'+value_received)

        data_tosend = getRequestedData(on_cols, off_cols, blacklist_cols, filtered_data, data_tosend, switch_received, value_received, column_received, dataframes, history_global, columns)

        return jsonify(data_tosend)
    else:
        history_global = [] # Empty the history global array because the system is refreshed
        filtered_data = data_dummy # Create a new dataframe to edit the values
        on_cols = list(column_dummy[1:]) # List of bars that are turned on. Initially, all the bars are on. We start from 1st index because column_dummy[0] = Throughput
        off_cols = [] # No bars are turned off initially
        '''
        In the next step, a new dataframe is created storing the variable categories and
        the Max and Min Throughput value for each category. This new dataframe is then
        converted to a JSON dictionary and sent to the client.
        '''
        cur_variable = on_cols[0].split('_')[0] # Used to store the variable part of the string, for ex: Workload_Data is a string them cur_variable will store "Workload".
        global_temp_df = pd.DataFrame(columns=[cur_variable, 'IsPresent', 'Max', 'Min', '10', '25', '50', '75', '90', 'Mean', 'Data']) # Dataframe used to store the categories and their Max and Min Throughput values
        for col in on_cols: # For all the bars that are currently in the "on" state
            temp = filtered_data.loc[filtered_data[col] == 1] # Get the dataframe where the current category for a given variable is 1 i.e. 'on'. This is used to plot individual bars.
            temp_var = col.split('_')[0] # Get the variable part of the string
            temp_cat = col.split('_')[1] # Get the category part of the string
            if temp_var == cur_variable: # There is no change in the variable of the col
                global_temp_df = global_temp_df.append({temp_var: temp_cat,
                                                        'IsPresent': 1,
                                                        'Max': temp.Throughput.max(),
                                                        'Min': temp.Throughput.min(),
                                                        '10': temp.Throughput.quantile(0.1),
                                                        '25': temp.Throughput.quantile(0.25),
                                                        '50': temp.Throughput.quantile(0.5),
                                                        '75': temp.Throughput.quantile(0.75),
                                                        '90': temp.Throughput.quantile(0.9),
                                                        'Mean': temp.Throughput.mean(),
                                                        'Data': list(temp.Throughput)}, ignore_index=True)
                # Append a row in the global_temp_df that includes the Min and Max, Lower Quantile, Upper Quantile and Median values for a new category of current variable
            else:
                '''
                Now we're ready to put the values of global_temp_df into dataframes dictionary. But before that, there's a user specification
                that needs to be taken care of. In the dataset, some values were named 'none' which are to be displayed before any other category for
                that variable. So, we bring the row containing 'none' as the category name to the top of the dataframe.

                Secondly, there are some categories that contain strings with numbers. For such categories, we need natural sorting instead of trivial
                string. For example, For Inode Size containing {128, 512, 256, 1024, none}, the correct sorted order should be {none, 128, 256, 512, 1024}. But since, we read
                the categories as strings, the sorted order is produced to be {1024, 128, 256, 512, none}. To sort such strings correctly, we use
                natural sorting along with explicitly adding 'none' to the start of the sorted dataframe.
                '''
                global_temp_df.set_index(list(global_temp_df.columns)[0], inplace=True) # Set the index of global_temp_df to the variable name, for example Workload, FileSystem etc.
                global_temp_df = global_temp_df.reindex(index=natsorted(global_temp_df.index)) # Sort the newly created index column based on natural sort
                global_temp_df['new'] = range(1,len(global_temp_df)+1) # Create a new column to give numbers to each of the rows in the dataframe
                if 'none' in global_temp_df.index: # If the current variable contains none, we need to bring that to the top of the dataframe
                    global_temp_df.at['none', 'new'] =  0 # Assign the number 0 to the row that contains none.
                    # Notice that we assigned numbers from range 1 to length of the dataframe to each row under 'new' column.
                    # As we assign 'none' containing row a value 0, if we sort the dataframe with column 'new', 'none' will come on top automatically.
                global_temp_df = global_temp_df.sort_values('new').drop('new', axis=1) # Sort the dataframe based on values in 'new' column to bring 'none' on top
                global_temp_df.reset_index(inplace=True) # Reset the index
                dataframes[cur_variable] = global_temp_df # Add the value of current dataframe to the dataframes dict for storage

                # Code to create a JSON Object start -------------

                chart_df = global_temp_df.to_dict(orient='records') # Convert the dataframe to a dict
                chart_df = json.dumps(chart_df) # Create a JSON object
                data_tosend[cur_variable] = chart_df # Add the JSON object to JSON array which is sent to the client

                # JSON object code end here --------------
                cur_variable = temp_var # A new variable is detected, so change the value of cur_variable to the new variable
                '''
                IsPresent in the dataframe is used to remember which are the bars that are actually affected by the current configuration selection.
                IsPresent = 1 if a category's throughput changes based on current configuration selection, on the other hand
                IsPresent = 0 if a category disappears from the dataset based on current configuration.

                For example: if the user selects {Workload: FileServer, SpecialOption: nodatasum}, in this case, many categories disappear as
                they are not possible to occur in reality or they are not present in the dataset. In such case, the bars for these categories should disappear from the plot.

                IsPresent stores this information and is used in index.js to make the bars appear/disappear.
                '''
                temp_df = pd.DataFrame(columns=[temp_var, 'IsPresent', 'Max', 'Min', '10', '25', '50', '75', '90', 'Mean', 'Data']) # Create a new temporary dataframe because a new variable type is detected
                temp_df = temp_df.append({temp_var: temp_cat,
                                        'IsPresent': 1,
                                        'Max': temp.Throughput.max(),
                                        'Min': temp.Throughput.min(),
                                        '10': temp.Throughput.quantile(0.1),
                                        '25': temp.Throughput.quantile(0.25),
                                        '50': temp.Throughput.quantile(0.5),
                                        '75': temp.Throughput.quantile(0.75),
                                        '90': temp.Throughput.quantile(0.9),
                                        'Mean': temp.Throughput.mean(),
                                        'Data': list(temp.Throughput)}, ignore_index=True)
                # Add the Max and Min throughput values to the temp dataframe for current category
                global_temp_df = temp_df # Change global_temp_df to this new dataframe for a new variable

        # At this point, global_temp_df contains the dataframe for the last variable in the dataset. We need to add this dataset to the JSON array data_tosend

        dataframes[cur_variable] = global_temp_df # Add the value of current dataframe to the dataframes dict for storage
        # Code to create a JSON Object start -------------

        chart_df = global_temp_df.to_dict(orient='records') # Convert the dataframe to a dict
        chart_df = json.dumps(chart_df) # Create a JSON object
        data_tosend[cur_variable] = chart_df # Add the JSON object to JSON array which is sent to the client

        # JSON object code end here --------------
        data_tosend['Max Thp'] = filtered_data.Throughput.max() # Max throughput of full data
        data_tosend['Min Thp'] = filtered_data.Throughput.min() # Min throughput of full data
        data_tosend['10 Thp'] = filtered_data.Throughput.quantile(0.1)
        data_tosend['25 Thp'] = filtered_data.Throughput.quantile(0.25)
        data_tosend['50 Thp'] = filtered_data.Throughput.quantile(0.5)
        data_tosend['75 Thp'] = filtered_data.Throughput.quantile(0.75)
        data_tosend['90 Thp'] = filtered_data.Throughput.quantile(0.9)
        data_tosend['Mean Thp'] = filtered_data.Throughput.mean()
        max_var_len = 0
        for col in columns:
            if len(col) > max_var_len:
                max_var_len = len(col)
        data_tosend['Max Var Len'] = max_var_len
        # pred_list = getPredictions(filtered_data, on_cols)
        # data_tosend['Pred List'] = pred_list
        # Sample the data to display on the frontend (This is to make the app run faster)
        if len(filtered_data.Throughput) > 20000:
            data_tosend['Data Thp'] = list(filtered_data.Throughput.sample(frac=0.3))
        else:
            data_tosend['Data Thp'] = list(filtered_data.Throughput)
        # History disctionary is added to history_global list to record a state change
        history_list = {"on_cols": list(on_cols), "off_cols": list(off_cols), "blacklist_cols": list(blacklist_cols), "Thp Max": filtered_data.Throughput.max(), "Thp Min": filtered_data.Throughput.min()}
        history_global.append(history_list)
        data_tosend['History'] = history_global
        return render_template('index.html', data=data_tosend)

if __name__=="__main__":
    '''
        This method is run only once when the server is started
    '''
    data = pd.read_csv('dataset/systems_data2.csv') # Reading the dataset is done only once when the server is started
    columns = list(data.columns) # Crete a list of columns in the dataset
    for col in columns[:-1]: # For all columns except throughput
        data[col] = data[col].apply(str) # Change the datatype for each column to be of type string so that there are no conflicts when performing calculations on each of the columns
    data_dummy = pd.get_dummies(data) #Since all the data in categorical, this creates a boolean dummy dataframe by creating columns of all the categories for each variable. This creates a column for each bar displayed
    column_dummy = list(data_dummy.columns)
    on_cols = [] # List of bars that are turned on. Initially, all the bars are on. We start from 1st index because column_dummy[0] = Throughput
    off_cols = [] # No bars are turned off initially
    data_tosend = {'columns': columns[:-1]}
    data_tosend['Max Cols'] = len(column_dummy)-1 # Total number of bars to display i.e. all columns in the dummy dataframe except throughput
    dataframes = {} # This variable stores all variables in the dataset and the statistics of throughput for each of these variables. It is a dict of dataframes
    blacklist_cols = [] # It stores the variables for which the buttons are turned 'off'.
    history_global = [] # List used to store on_cols, off_cols and blacklist_cols for each state. Used for blockchain plot
    app.run(debug=True)
    # app.run(host='0.0.0.0', threaded=True)


# -*- coding: utf-8 -*-
"""
Created on Thu Apr  5 18:36:02 2018

@author: mchini
"""

import scipy.io as scio
import numpy as np
from sklearn.feature_selection import SelectPercentile, mutual_info_regression
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler, RobustScaler, quantile_transform, QuantileTransformer
from sklearn.neural_network import MLPRegressor
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import train_test_split
from sklearn.decomposition import PCA

depth = np.sort(np.tile(np.arange(1, 5), 15)) - 1
data = scio.loadmat("Q:/Personal/Mattia/Python/classifiers/anesthesia/database_full")['features']

y = data[:, 0]
X = data[:, 1:]

num_features = 22

time = np.arange(1, 61)
time = np.tile(time, np.int(len(y) / len(time)))
#time_conf_matrix = np.zeros((1, 60))

classifier = MLPRegressor(solver='lbfgs', activation='relu')
parameters = {'hidden_layer_sizes':[(50,30), (30,30)], 'alpha':[0.5, 3, 5, 10]}
# 'activation':('identity', 'logistic', 'tanh', 'relu')
# 'alpha':[0.5, 1, 3, 5]

qnt_trf = QuantileTransformer(n_quantiles=100000)
X = qnt_trf.fit_transform(X)
#pca = PCA(n_components = 22, whiten=True)
#X = pca.fit_transform(X)
#X = StandardScaler().fit_transform(X)

nboots = 10
score = np.zeros((1, nboots))
depth_matrix = np.tile(depth, (nboots, 1))

idx = 0

best_parameters = [dict() for x in np.arange(0, nboots)]

predictions = np.zeros((nboots, max(time)))

for nboot in range(0, nboots):

    X_train, X_test, y_train, y_test, time_train, time_test = \
        train_test_split(X, y, time, test_size = .20)

    selector = SelectPercentile(mutual_info_regression)
    selector.fit(X_train, y_train)
    scores = selector.scores_
    scores /= scores.max()
    features = scores.argsort()[-num_features:][::-1]

    classifiers = GridSearchCV(classifier, parameters) # , scoring='neg_median_absolute_error'
    classifiers.fit(X_train[:, features], y_train)
    best_parameters[nboot] = classifiers.best_params_
    classifiers = classifiers.best_estimator_
    classifier.fit(X_train[:, features], y_train)
    score[0, nboot] = classifier.score(X_test[:, features], y_test)

    predictions[nboot, time_test - 1] = classifier.predict(X_test[:, features])

    #mistakes = predictions == y_test
    #time_mistakes = time_test[mistakes == False]
    #time_conf_matrix[0, time_mistakes - 1] = time_conf_matrix[0, time_mistakes - 1] + 1

predictions[predictions==0] = np.nan
predictions = predictions - 1
accuracy = np.abs(predictions - depth_matrix)

plt.figure()
plt.scatter(np.where(predictions)[1], predictions, alpha=0.1)
plt.plot(depth, c='r')
plt.errorbar(np.arange(0, max(time)), np.nanmean(predictions, 0), np.nanstd(predictions, 0) / np.sqrt(nboots), xerr=None, c='y')
plt.title('prediction of anesthetic concentration - R2 = ' + '{:.5}'.format(str(np.mean(score))))
plt.xlabel('time (minutes)')
plt.ylabel('anesthesia depth')

plt.figure()
plt.scatter(np.where(accuracy)[1], accuracy, alpha=0.1)
plt.errorbar(np.arange(0, max(time)), np.nanmedian(accuracy, 0), np.nanstd(accuracy, 0) / np.sqrt(nboots), xerr=None, c='r')
plt.title('median prediction error (% of anesthetic) - error = ' + '{:.5}'.format(str(np.nanmedian(accuracy))))
plt.xlabel('time (minutes)')
plt.ylabel('prediction error')

plt.show()

anesthesia/classifiers scripts (old stuff)/classifier_v3.py


import re
import json
import requests
from pprint import pprint

def main():
    url = 'http://www.acfun.cn/comment_list_json.aspx?contentId=3491484&currentPage=1'
    r = requests.get(url, verify=False)
    mjson = json.loads(r.text)
    num = 0
    print(mjson['data'])
    commentContentArr = mjson['data']['commentContentArr']
    for var in commentContentArr:
        #print('')
        num = num+1
        print(commentContentArr[var]['userName']+" "+commentContentArr[var]['content'])
    print(num)
if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='Item',
            fields=[
                ('id', models.AutoField(auto_created=True, verbose_name='ID', serialize=False, primary_key=True)),
                ('count', models.IntegerField()),
            ],
        ),
        migrations.CreateModel(
            name='Menu',
            fields=[
                ('id', models.AutoField(auto_created=True, verbose_name='ID', serialize=False, primary_key=True)),
                ('name', models.CharField(max_length=1000, unique=True)),
                ('description', models.CharField(max_length=1000)),
                ('price', models.FloatField(default=0.0)),
                ('created', models.DateTimeField(auto_now_add=True)),
            ],
            options={
                'ordering': ('created',),
            },
        ),
        migrations.CreateModel(
            name='Order',
            fields=[
                ('id', models.AutoField(auto_created=True, verbose_name='ID', serialize=False, primary_key=True)),
                ('order_date', models.DateTimeField(auto_now=True)),
                ('delivered', models.BooleanField(default=False)),
            ],
            options={
                'ordering': ('order_date',),
            },
        ),
        migrations.CreateModel(
            name='Shop',
            fields=[
                ('id', models.AutoField(auto_created=True, verbose_name='ID', serialize=False, primary_key=True)),
                ('name', models.CharField(max_length=1000, unique=True)),
                ('owner', models.CharField(max_length=1000, unique=True)),
                ('created', models.DateTimeField(auto_now_add=True)),
            ],
            options={
                'ordering': ('created',),
            },
        ),
        migrations.CreateModel(
            name='User',
            fields=[
                ('id', models.AutoField(auto_created=True, verbose_name='ID', serialize=False, primary_key=True)),
                ('name', models.CharField(max_length=1000, unique=True)),
                ('created', models.DateTimeField(auto_now_add=True)),
            ],
            options={
                'ordering': ('created',),
            },
        ),
        migrations.AddField(
            model_name='order',
            name='shop',
            field=models.ForeignKey(to='order.Shop'),
        ),
        migrations.AddField(
            model_name='order',
            name='user',
            field=models.ForeignKey(to='order.User'),
        ),
        migrations.AddField(
            model_name='menu',
            name='shop',
            field=models.ForeignKey(to='order.Shop'),
        ),
        migrations.AddField(
            model_name='item',
            name='menu',
            field=models.ForeignKey(to='order.Menu'),
        ),
        migrations.AddField(
            model_name='item',
            name='order',
            field=models.ForeignKey(to='order.Order', related_name='items'),
        ),
    ]


import os
from PyQt5.QtCore import QStringListModel, QModelIndex, QItemSelectionModel, Qt
from PyQt5.QtWidgets import QListView

FILES_PATH = "/".join(
    os.path.join(os.path.dirname(os.path.abspath(__file__)).split('/')[:-1])) + '/data/'

class PathsListView(QListView):
    """List all .csv files contained in the folder '../dados/'"""

    def __init__(self, parent=None):

        QListView.__init__(self, parent)

        self._model = QStringListModel()
        self.fetch_paths()
        self.setModel(self._model)
        self.setWindowTitle('PathsList')
        self.setFocusPolicy(Qt.ClickFocus)
        self.clicked[QModelIndex].connect(self.item_clicked)

        self.selectionModel().select(
            self._model.createIndex(0, 0), QItemSelectionModel.Select)

    def fetch_paths(self):
        """Fetch all valid paths to .csv files and display them"""

        pathList = [path for path in os.listdir(
            FILES_PATH) if path.endswith('.csv') and int(path[0]) > 3]
        pathList.sort(reverse=True)

        for i, path in enumerate(pathList):

            pathList[i] = path + ' - ' + str(
                round(float(os.path.getsize(FILES_PATH + path)) / pow(10, 6), 2)) + ' mb'

        self._model.setStringList(pathList)

    def get_path(self):
        """Get path of currently selected file

        Return:
            path (string): path of the current file
        """

        return FILES_PATH + self.selectedIndexes()[0].data().split(' ')[0]

    def item_clicked(self, index):
        """
        Check if step and number values, read GUI.misc_menu documentation, are valid.

        Inca se they aren't change them to default [1, 1]
        """

        values = [int(s)
                  for s in index.data().replace('.', '_').split('_') if s.isdigit()][2:]

        if len(values) != 2:
            values = [1, 1]

        self.parent().parent().set_san_values(values)

if __name__ == '__main__':
    pass


#import pyserial module for communication between computer and arduino
import serial
#import pyautogui module to trigger macros/keyboard commands
import pyautogui

#open a serial port for the Arduino
arduino = serial.Serial('/dev/tty.usbserial-DA00SR57', 9600)

#set up an infinite loop
while True:
    #readline() method reads a string from opened serial port until
    #'\n' escape character is detected
    output_str = arduino.readline()

    #chain of elif's compare received string to known strings
    #if a known string is detected, a specific pyautoui method
    #is executed for scrolling or tab switching

    #if a string with '1' and terminated with '\n' is received...
    if(output_str == '1\n'):
        #scroll up by '1' "click" (a unit of scroll movement)
        pyautogui.scroll(1)
    elif(output_str == '2\n'):
        pyautogui.scroll(3)
    elif(output_str == '3\n'):
        pyautogui.scroll(5)
    #if a string with '-1' and terminated with '\n' is received...
    elif(output_str == '-1\n'):
        #scroll by '-1' "clicks" (scroll downwards, in other words)
        pyautogui.scroll(-1)
    elif(output_str == '-2\n'):
        pyautogui.scroll(-3)
    elif(output_str == '-3\n'):
        pyautogui.scroll(-5)
    #if a string with 'left' and '\n' is received...
    elif(output_str == 'left\n'):
        #execute the macro to switch tabs in the leftward direction
        pyautogui.hotkey('ctrl', 'pageup')
    #if a string with 'right' and '\n' is received...
    elif(output_str == 'right\n'):
        #execute the macro to switch ttabs in the rightward direction
        pyautogui.hotkey('ctrl', 'pagedown')


# 离散特征处理

import pandas as pd
df = pd.DataFrame([
            ['green', 'M', 10.1, 'class1'],
            ['red', 'L', 13.5, 'class2'],
            ['blue', 'XL', 15.3, 'class1']])

df.columns = ['color', 'size', 'prize', 'class label']

class_mapping = {label:idx for idx,label in enumerate(set(df['class label']))}

df['class label'] = df['class label'].map(class_mapping)

size_mapping = {'XL': 3, 'L': 2, 'M': 1}

df['size'] = df['size'].map(size_mapping)

color_mapping = {'green': (0, 0, 1), 'red': (0, 1, 0), 'blue': (1, 0, 0)}

df['color'] = df['color'].map(color_mapping)

inv_color_mapping = {v: k for k, v in color_mapping.items()}
inv_size_mapping = {v: k for k, v in size_mapping.items()}
inv_class_mapping = {v: k for k, v in class_mapping.items()}

df['color'] = df['color'].map(inv_color_mapping)
df['size'] = df['size'].map(inv_size_mapping)
df['class label'] = df['class label'].map(inv_class_mapping)

from sklearn.preprocessing import LabelEncoder

class_le = LabelEncoder()
df['class label'] = class_le.fit_transform(df['class label'])

class_le.inverse_transform(df['class label'])

df.transpose().to_dict().values()

feature = df.iloc[:, :-1]

from sklearn.feature_extraction import DictVectorizer
dvec = DictVectorizer(sparse=False)

X = dvec.fit_transform(feature.transpose().to_dict().values())

pd.DataFrame(X, columns=dvec.get_feature_names())

color_le = LabelEncoder()
df['color'] = color_le.fit_transform(df['color'])

from sklearn.preprocessing import OneHotEncoder
ohe = OneHotEncoder(sparse=False)

X = ohe.fit_transform(df[['color']].values)

import pandas as pd
df = pd.DataFrame([
            ['green', 'M', 10.1, 'class1'],
            ['red', 'L', 13.5, 'class2'],
            ['blue', 'XL', 15.3, 'class1']])

df.columns = ['color', 'size', 'prize', 'class label']

size_mapping = {'XL': 3, 'L': 2, 'M': 1}
df['size'] = df['size'].map(size_mapping)

class_mapping = {label:idx for idx,label in enumerate(set(df['class label']))}
df['class label'] = df['class label'].map(class_mapping)

pd.get_dummies(df)

# 特征标准化

import pandas as pd
import numpy as np

df = pd.read_csv(
    '../data/wine_data.csv',  #葡萄酒数据集
     header=None,     #用哪行当做列名，我们自己来指定
     usecols=[0,1,2]  #返回一个子集，我们拿部分特征举例就可以了
    )

df.columns=['Class label', 'Alcohol', 'Malic acid']

df.head()

from sklearn import preprocessing

std_scale = preprocessing.StandardScaler().fit(df[['Alcohol', 'Malic acid']])
df_std = std_scale.transform(df[['Alcohol', 'Malic acid']])

minmax_scale = preprocessing.MinMaxScaler().fit(df[['Alcohol', 'Malic acid']])
df_minmax = minmax_scale.transform(df[['Alcohol', 'Malic acid']])

print('Mean after standardization:\nAlcohol={:.2f}, Malic acid={:.2f}'
      .format(df_std[:,0].mean(), df_std[:,1].mean()))
print('\nStandard deviation after standardization:\nAlcohol={:.2f}, Malic acid={:.2f}'
      .format(df_std[:,0].std(), df_std[:,1].std()))

print('Min-value after min-max scaling:\nAlcohol={:.2f}, Malic acid={:.2f}'
      .format(df_minmax[:,0].min(), df_minmax[:,1].min()))
print('\nMax-value after min-max scaling:\nAlcohol={:.2f}, Malic acid={:.2f}'
      .format(df_minmax[:,0].max(), df_minmax[:,1].max()))

from matplotlib import pyplot as plt

def plot():
    plt.figure(figsize=(8, 6))

    plt.scatter(df['Alcohol'], df['Malic acid'],
                color='green', label='input scale', alpha=0.5)

    plt.scatter(df_std[:, 0], df_std[:, 1], color='red',
                label='Standardized [$N  (\mu=0, \; \sigma=1)$]', alpha=0.3)

    plt.scatter(df_minmax[:, 0], df_minmax[:, 1],
                color='blue', label='min-max scaled [min=0, max=1]', alpha=0.3)

    plt.title('Alcohol and Malic Acid content of the wine dataset')
    plt.xlabel('Alcohol')
    plt.ylabel('Malic Acid')
    plt.legend(loc='upper left')
    plt.grid()

    plt.tight_layout()

plot()
plt.show()

fig, ax = plt.subplots(3, figsize=(6, 14))

for a, d, l in zip(range(len(ax)),
                   (df[['Alcohol', 'Malic acid']].values, df_std, df_minmax),
                   ('Input scale',
                    'Standardized [$N  (\mu=0, \; \sigma=1)$]',
                    'min-max scaled [min=0, max=1]')
                   ):
    for i, c in zip(range(1, 4), ('red', 'blue', 'green')):
        ax[a].scatter(d[df['Class label'].values == i, 0],
                      d[df['Class label'].values == i, 1],
                      alpha=0.5,
                      color=c,
                      label='Class %s' % i
                      )
    ax[a].set_title(l)
    ax[a].set_xlabel('Alcohol')
    ax[a].set_ylabel('Malic Acid')
    ax[a].legend(loc='upper left')
    ax[a].grid()

plt.tight_layout()

plt.show()

import pandas as pd

df = pd.read_csv(
    '../data/wine_data.csv',
    header=None,
    )
df.head()

from sklearn.model_selection import train_test_split

X_wine = df.values[:,1:]
y_wine = df.values[:,0]

X_train, X_test, y_train, y_test = train_test_split(X_wine, y_wine,
    test_size=0.30, random_state=12345)

from sklearn import preprocessing

std_scale = preprocessing.StandardScaler().fit(X_train)
X_train_std = std_scale.transform(X_train)
X_test_std = std_scale.transform(X_test)

from sklearn.decomposition import PCA

# on non-standardized data
pca = PCA(n_components=2).fit(X_train)
X_train = pca.transform(X_train)
X_test = pca.transform(X_test)

# om standardized data
pca_std = PCA(n_components=2).fit(X_train_std)
X_train_std = pca_std.transform(X_train_std)
X_test_std = pca_std.transform(X_test_std)

from matplotlib import pyplot as plt

fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(10, 4))

for l, c, m in zip(range(1, 4), ('blue', 'red', 'green'), ('^', 's', 'o')):
    ax1.scatter(X_train[y_train == l, 0], X_train[y_train == l, 1],
                color=c,
                label='class %s' % l,
                alpha=0.5,
                marker=m
                )

for l, c, m in zip(range(1, 4), ('blue', 'red', 'green'), ('^', 's', 'o')):
    ax2.scatter(X_train_std[y_train == l, 0], X_train_std[y_train == l, 1],
                color=c,
                label='class %s' % l,
                alpha=0.5,
                marker=m
                )

ax1.set_title('Transformed NON-standardized training dataset after PCA')
ax2.set_title('Transformed standardized training dataset after PCA')

for ax in (ax1, ax2):
    ax.set_xlabel('1st principal component')
    ax.set_ylabel('2nd principal component')
    ax.legend(loc='upper right')
    ax.grid()
plt.tight_layout()

plt.show()


import discord
from discord import app_commands
from discord.ext import commands

from typing import Optional
from random import choice, randint

from lxml import html
from PIL import Image
from io import BytesIO

from credentials import DEBUG_CHANNEL

class Card(commands.Cog,
    command_attrs={"cooldown": commands.CooldownMapping.from_cooldown(
        1, 30, commands.BucketType.user)}):

    def __init__(self, bot):
        self.bot = bot

    @commands.hybrid_command()
    @app_commands.describe(game="TCG you want to pull a card from")
    async def card(self, ctx, game: Optional[str] = None):
        """ Pulls a random TCG card """

        if game:
            if game.startswith("poke"):
                await self.pokemon(ctx)
            elif game.startswith(("yugioh", "ygo")):
                await self.yugioh(ctx)
            elif game.startswith(("mtg", "magic")):
                await self.mtg(ctx)
            elif game.startswith("digi"):
                await self.digimon(ctx)
            elif game.startswith(("flesh", "fab")):
                await self.fleshandblood(ctx)
            elif game.startswith("gateruler"):
                await self.gateruler(ctx)
            elif game.startswith(("finalfantasy", "fftcg")):
                await self.finalfantasy(ctx)
            elif game.startswith(("cardfightvanguard", "cfv")):
                await self.cardfightvanguard(ctx)
            elif game.startswith("grandarchive"):
                await self.grandarchive(ctx)
            elif game.startswith("nostalg"):
                await self.nostalgix(ctx)
            else:
                command = choice(self.get_commands())
                await command.__call__(ctx)
        else:
            command = choice(self.get_commands())
            await command.__call__(ctx)

    @card.autocomplete('game')
    async def card_autocomplete(self,
        interaction: discord.Interaction,
        current: str,) -> list[app_commands.Choice[str]]:

        games = ['pokemon', 'yugioh', 'magic', 'digimon',
            'fleshandblood', 'gateruler', 'finalfantasy',
            'cardfightvanguard', 'grandarchive', 'nostalgix']

        return [app_commands.Choice(name=game, value=game)
            for game in games if current.lower() in game.lower() ]

    @commands.command(aliases=['poke'])
    async def pokemon(self, ctx):
        """ Pulls a random Pokemon TCG card """

        url = "https://pkmncards.com/?random"
        r = await self.bot.http_client.get(url, follow_redirects=True)

        # Scrape image from page metadata
        page = html.fromstring(r.text)
        image_url = page.xpath("//meta[@property='og:image']/@content")[0]
        image_url = image_url.split('?')[0]

        await ctx.send(image_url)

    @commands.command(aliases=['ygo'])
    async def yugioh(self, ctx):
        """ Pulls a random Yu-Gi-Oh! card """

        url = "https://db.ygoprodeck.com/api/v7/randomcard.php"
        r = await self.bot.http_client.get(url)
        card = r.json()

        await ctx.send(card['card_images'][0]['image_url'])

    @commands.command()
    async def digimon(self, ctx):
        """ Pulls a random Digimon card. """

        url = ("https://digimoncard.io/api-public/"
            "getAllCards.php?series=Digimon%20Card%20Game")
        r = await self.bot.http_client.get(url)
        card_list = r.json()

        await ctx.send(("https://images.digimoncard.io/images/cards/"
            f"{choice(card_list)['cardnumber']}.jpg"))

    @commands.command(aliases=['magic'])
    async def mtg(self, ctx):
        """ Pulls a random Magic the Gathering card """

        url = "https://api.scryfall.com/cards/random"
        r = await self.bot.http_client.get(url)
        card = r.json()

        # If it doesn't have an image then try again
        card_status = card['image_status']
        if card_status == 'missing' or card_status == 'placeholder':
            await self.mtg(ctx)
        else:
            # If card has two sides, pick one side
            if not "image_uris" in card and "card_faces" in card:
                card['image_uris'] = card['card_faces'][randint(0,1)]['image_uris']

            await ctx.send(card['image_uris']['border_crop'])

    @commands.command(aliases=['fab'])
    async def fleshandblood(self, ctx):
        """ Pulls a random Flesh and Blood card """

        url = "https://api.fabdb.net/cards"
        r = await self.bot.http_client.get(url)
        page = r.json()

        # Pick random page
        page_select = randint(1, page['meta']['last_page'])
        params = { "page": page_select }

        # Get random card from random page
        r = await self.bot.http_client.get(url, params=params)
        page = r.json()
        card = choice(page['data'])

        await ctx.send(card['image'].split('?')[0])

    @commands.command()
    async def gateruler(self, ctx):
        """ Pulls a random Gate Ruler card """

        # Defer in case edit takes too long
        await ctx.defer()

        # Get max page number
        url = "https://www.gateruler-official.com/card_search"
        r = await self.bot.http_client.get(url)
        page = html.fromstring(r.text)

        max_pages = page.xpath("//ul[@class='pagination']/li/a/text()")[-2]

        # Pick random page
        params = { "page": randint(1, int(max_pages)) }
        r = await self.bot.http_client.get(url, params=params)
        page = html.fromstring(r.text)

        # Get random card
        cards = page.xpath("//li[@class='com_btm']/a/img/@src")
        card_url = choice(cards)

        r = await self.bot.http_client.get(card_url)

        # Crop borders of card
        card_img = Image.open(BytesIO(r.content))
        card_img = card_img.crop(card_img.getbbox())

        # Send to Discord
        with BytesIO() as img_binary:
            card_img.save(img_binary, 'PNG')
            img_binary.seek(0)
            await ctx.send(file=discord.File(
                fp=img_binary,
                filename=card_url.rsplit('/', 1)[1]))

    @commands.command(aliases=['fftcg'])
    async def finalfantasy(self, ctx):
        """ Pulls a random Final Fantasy TCG card """

        # Defer in case JSON download takes too long
        await ctx.defer()

        url = "https://fftcg.square-enix-games.com/na/get-cards"
        r = await self.bot.http_client.get(url)
        cards = r.json()

        card = choice(cards['cards'])
        if '\/' in card['code']:
            card['code'] = card['code'].split('\/')[0]

        await ctx.send(("https://fftcg.cdn.sewest.net/images/cards/"
            f"full/{card['code']}_eg.jpg"))

    @commands.command(aliases=["cfvangaurd", "cfv"])
    async def cardfightvanguard(self, ctx):
        """ Pulls a random Cardfight!! Vanguard card """

        # Defer in case multiple requests take too long
        await ctx.defer()

        # Get first page to figure out max page
        url = "https://en.cf-vanguard.com/cardlist/cardsearch"
        r = await self.bot.http_client.get(url)
        page = html.fromstring(r.text)

        # 24 cards per page
        card_count = page.xpath("//div[@class='number']/text()")[0]
        card_count = int(card_count[:-8])
        params = {
            "page": randint(1, int((card_count / 24) + 1))
        }

        # Get page with cards to pick
        r = await self.bot.http_client.get(url, params=params)
        page = html.fromstring(r.text)

        # Pick card
        card = "https://en.cf-vanguard.com{}".format(
            choice(page.xpath("//img[@class='object-fit-img']/@src")))
        await ctx.send(card)

    @commands.command()
    async def grandarchive(self, ctx):
        """ Pulls a random Grand Archive card """

        r = await self.bot.http_client.get(
            "https://api.gatcg.com/cards/random?amount=1")
        card = r.json()[0]
        card_slug = choice(card["editions"])["slug"]

        await ctx.send(f"https://ga-index-public.s3.us-west-2.amazonaws.com/cards/{card_slug}.jpg")

    @commands.command()
    async def nostalgix(self, ctx):
        """ Pulls a random Nostalgix card """

        url = "https://play-api.carde.io/v1/cards/63bc844c3e8d2f34e312bc77"

        r = await self.bot.http_client.get(url)
        max_pages = r.json()["pagination"]["totalPages"]

        params = { "page": randint(1, int(max_pages)) }

        r = await self.bot.http_client.get(url, params=params)
        cards = r.json()["data"]

        await ctx.send(choice(cards)["imageUrl"])

async def setup(bot):
    await bot.add_cog(Card(bot))


import nest
from matplotlib import pyplot as plt
import numpy as np

syn_ex = {"weight":1.2}
syn_in = {"weight":-2.0}

neuron = nest.Create("iaf_psc_alpha")
voltmeter = nest.Create("multimeter")
spike_detector = nest.Create("spike_detector",params={"withtargetgid":True,"to_file":False,"label":"spike-detector-out"})
noise_ex = nest.Create("poisson_generator")
noise_in = nest.Create("poisson_generator")

nest.SetStatus(neuron,{"I_e":0.})
nest.SetStatus(voltmeter,{"withtime":True,"record_from":["V_m"]})
nest.SetStatus(noise_ex,{"rate":80000.})
nest.SetStatus(noise_in,{"rate":15000.})

nest.Connect(noise_ex,neuron,syn_spec=syn_ex)
nest.Connect(noise_in,neuron,syn_spec=syn_in)
nest.Connect(voltmeter,neuron)
nest.Connect(neuron,spike_detector)

nest.Simulate(1000.0)

dmm = nest.GetStatus(voltmeter,keys="events")[0]
# print(neuron)
# print(dmm)
Vms = dmm["V_m"]
max_Vms = np.max(Vms)
ts = dmm["times"]

plt.figure(1)
plt.plot(ts,Vms)

dmm2 = nest.GetStatus(spike_detector)[0]
spike_times = dmm2["events"]["times"]
# plt.figure(2)
# plt.subplot(212)
plt.plot(spike_times,np.ones(len(spike_times))*max_Vms,'bo')
print(dmm2)
plt.show()


from datetime import datetime, timezone

def current_time():
    """
    Convert time now to desired format.
    """
    now = datetime.now(timezone.utc)
    date_format = "%a, %d %b %Y %H:%M:%S GMT"
    time = now.strftime(date_format)
    return time


# -*- coding: utf-8 -*-
"""
Created on Thu Apr 12 13:54:19 2018

@author: Ayoub El khallioui
"""
import numpy as np
import pandas as pd
from random import shuffle
import matplotlib.pyplot as plt
plt.style.use('ggplot')
class ANN(object):

    def __init__(self,eta=0.01,n_iter=40,n_hidden=30,size_minibatch=100,random_state=1,shuffle=True,l2=0.):
        self.eta=eta
        self.n_iter=n_iter
        self.random_state=random_state
        self.shuffle=shuffle
        self.l2=l2
        self.n_hidden=n_hidden
        self.size_minibatch=size_minibatch

    def _oneHotEncoder(self,y):
        return pd.get_dummies(y).values

    def fit(self,X,y):
        gen=np.random.RandomState(self.random_state)
        self.wh=gen.normal(loc=0.0,scale=0.1,size=(X.shape[1]+1,self.n_hidden))
        self.wout=gen.normal(loc=0.0,scale=0.1,size=(self.n_hidden+1,np.unique(y).shape[0]))
        self.cost_=[]
        y_enc=self._oneHotEncoder(y)
        for _ in range(self.n_iter):
            indices=np.arange(X.shape[0])
            if self.shuffle:
                shuffle(indices)

            for i in range(0,indices.shape[0]-self.size_minibatch+1,self.size_minibatch):
                batch=indices[i:i+self.size_minibatch]
                aout,zout,ah,zh=self._forward_prop(X[batch])
                sigma_out=aout-y_enc[batch]
                sigma_h=np.dot(sigma_out,self.wout[1:].T)*self._derivative_sigmoid(ah)
                gradw_out=ah.T.dot(sigma_out)
                gradb_out=sigma_out.sum(axis=0)
                gradw_h=X[batch].T.dot(sigma_h)
                gradb_h=sigma_h.sum(axis=0)

                delta_out=gradw_out+self.l2*self.wout[1:]
                delta_h=gradw_h+self.l2*self.wh[1:]
                self.wh[1:]-=self.eta*delta_h
                self.wh[0]-=self.eta*gradb_h
                self.wout[1:]-=self.eta*delta_out
                self.wout[0]-=self.eta*gradb_out

            aout,zout,ah,zh=self._forward_prop(X)
            self.cost_.append(self._cost_function(aout,y_enc))
        return self

    def _sigmoid_activation(self,z):
        return 1/(1+np.exp(-np.clip(z,-260,260)))

    def _derivative_sigmoid(self,z):
        return z*(1-z)

    def _forward_prop(self,X):
        zh=np.dot(X,self.wh[1:])+self.wh[0]
        ah=self._sigmoid_activation(zh)
        zout=np.dot(ah,self.wout[1:])+self.wout[0]
        aout=self._sigmoid_activation(zout)
        return aout,zout,ah,zh

    def predict(self,X):
        aout,zout,ah,zh=self._forward_prop(X)
        return np.argmax(zout,axis=1)

    def _cost_function(self,output,y_enc):
        L2=self.l2*(np.sum(self.wh[1:]**2)+np.sum(self.wout[1:]**2))
        terme1=-y_enc*np.log(output)
        terme2=-(1-y_enc)*np.log(1-output)
        return np.sum(terme1+terme2)+L2

from sklearn.metrics import accuracy_score
mnist=np.load('D:\\python pyzo\\MNN from scratch\\mnist_scaled.npz')
files=mnist.files
X_train,y_train,X_test,y_test=[mnist[f] for f in files]

ANN=ANN(eta=0.001,n_iter=150,n_hidden=30,size_minibatch=100,random_state=1,shuffle=True,l2=0.01)
ANN.fit(X_train,y_train)
plt.plot(np.arange(1,len(ANN.cost_)+1),ANN.cost_,marker='o',color='black')
plt.xlabel('epochs')
plt.ylabel('cost_function')
plt.show()
y_pred=ANN.predict(X_test)
print("accuracy_score=%.3f"%accuracy_score(y_test,y_pred))


import time
import threading
import os
import shutil
import datetime
import traceback
import importlib
import subprocess

class HardwareEngine(object):
    STATUS_IDLE = "IDLE"
    STATUS_TESTING = "TESTING"

    # file storage
    file_folder = os.path.join('.', 'uploads')
    backup_root_folder = os.path.join('.', 'upload_backups')

    # testbed info
    status = STATUS_IDLE
    config = None

    # hardware info
    hardware_dict = None
    hardware_processing_order = None

    # hardware execution
    execution_threads = None

    # grading status
    task_secret_code = None
    task_running = None
    task_execution_time_sec = None
    aborting_task_timer = None

    def __init__(self, config):
        self.config = config
        hardware_dict = config['hardware_list']

        # get hardware initialization order
        if not 'hardware_processing_order' in config:
            self.hardware_processing_order = []
            for hardware_name in config['hardware_list']:
                self.hardware_processing_order.append(hardware_name)
        else:
            self.hardware_processing_order = config['hardware_processing_order']
            if len(self.hardware_processing_order) != len(config['hardware_list']):
                raise Exception('hardware_list and hardware_processing_order do not match in configuration file')
            for hardware_name in self.hardware_processing_order:
                if hardware_name not in config['hardware_list']:
                    raise Exception('hardware_list and hardware_processing_order do not match in configuration file')

        self.hardware_dict = {}
        for hardware_name in config['hardware_list']:
            hardware_config = config['hardware_list'][hardware_name]
            init_params = hardware_config['init_params']
            module_name, class_name = hardware_config['class'].rsplit(".", 1)
            MyClass = getattr(importlib.import_module(module_name), class_name)
            instance = MyClass(hardware_name, init_params, self, self.file_folder)
            self.hardware_dict[hardware_name] = instance

        for input_file in config['required_input_files']:
            if input_file in config['required_output_files']:
                raise Exception('required_input_files and required_output_files are overlapped')

        # prepare backup upload folder
        if not os.path.isdir(self.backup_root_folder):
            os.makedirs(self.backup_root_folder)

    def add_http_client(self, client):
        self.http_client = client

    def get_status(self):
        return self.status

    def reset_devices(self):
        for hardware_name in self.hardware_processing_order:
            self.hardware_dict[hardware_name].on_before_execution()

    def start_test(self):
        # notify all hardware the execution just begins
        self.execution_threads = []
        for hardware_name in self.hardware_processing_order:
            thread_name = 'Exe-%s' % hardware_name
            print('start_test', thread_name)
            t = threading.Thread(name=thread_name,
                    target=self.hardware_dict[hardware_name].on_execute)
            t.start()
            self.execution_threads.append(t)

    def notify_terminate(self):
        self._terminate_hardware_procedure()

    def _terminate_hardware_procedure(self):
        if not self.task_running:
            return

        self.task_running = False

        print(self)
        print(self.aborting_task_timer)
        self.aborting_task_timer.cancel()

        # send terminate signal to all hardware
        for hardware_name in self.hardware_processing_order:
            print('terminate', hardware_name)
            self.hardware_dict[hardware_name].on_terminate()

        for th in self.execution_threads:
            print('wait for', th)
            th.join()

        output_files = {}
        for file_name in self.config['required_output_files']:
            file_path = os.path.join(self.file_folder, file_name)
            output_files[file_name] = file_path

        if self.http_client.send_dut_output(output_files, self.task_secret_code):
            print('Waveform file uploaded')
        else:
            print('Unable to upload output to server')

        # send clean signal to all hardware
        for hardware_name in self.hardware_processing_order:
            self.hardware_dict[hardware_name].on_reset_after_execution()

        # backup
        now = datetime.datetime.now().strftime('%Y-%m-%d.%H:%M:%S.%f')
        task_backup_folder = os.path.join(self.backup_root_folder, now)
        os.makedirs(task_backup_folder)
        shutil.move(self.file_folder, task_backup_folder)

        # all tasks are done. update status
        self.status = self.STATUS_IDLE
        print('Test complete.')

        # update status over HTTP
        if self.http_client.send_tb_status(self.status):
            print('IDLE status sent')
        else:
            print('Unable to post status to server')

    def _grade_thread(self):
        print('_grade_thread')
        self.reset_devices()
        self.task_running = True
        self.start_test()
        self.aborting_task_timer = threading.Timer(
                self.task_execution_time_sec, self._terminate_hardware_procedure)
        print('_grade_thread', self.aborting_task_timer)
        self.aborting_task_timer.start()

    def request_grade_assignment(self, input_files, secret_code, execution_time_sec):
        """
        Return:
          True if succesfully storing the data
        """
        if self.status != self.STATUS_IDLE:
            return False

        self.status = self.STATUS_TESTING

        # store assignment info
        if not os.path.isdir(self.file_folder):
            os.makedirs(self.file_folder)
        subprocess.call(['rm', '-rf', '%s/*' % self.file_folder])
        for file_name in input_files:
            file_path = os.path.join(self.file_folder, file_name)
            with open(file_path, 'wb') as fo:
                fo.write(input_files[file_name])
        self.task_secret_code = secret_code
        self.task_execution_time_sec = execution_time_sec if execution_time_sec else 600

        print('request_grade_assignment')
        # start the grading task asynchronously
        threading.Thread(target=self._grade_thread, name=('id=%s' % self.config['id'])).start()

        return True


from flask import request
from flask_restplus import Resource

from ..dtos.user_dto import UserDto
from ..services.user_service import UserService

api = UserDto.api
_user = UserDto.user

@api.route('/')
class UserList(Resource):
    @api.doc('list_of_registered_users')
    @api.marshal_list_with(_user, envelope='data')
    def get(self):
        """List all registered users"""
        return UserService.get_all_users()

    @api.response(201, 'User successfully created.')
    @api.response(209, 'User failed to create. Username already taken.')

    @api.doc('create a new user')
    @api.expect(_user, validate=True)
    def post(self):
        """Creates a new User """
        data = request.json
        return UserService.save_new_user(data=data)

@api.route('/<username>')
@api.param('username', 'The User identifier')
@api.response(404, 'User not found.')
class User(Resource):
    @api.doc('get a user')
    @api.marshal_with(_user)
    def get(self, username):
        """get a user given its identifier"""
        user = UserService.get_a_user(username)
        if not user:
            api.abort(404, "User not found.")
        else:
            return user


from django import template
from reportedPhotos.models import ReportedPhotos
register = template.Library()

@register.filter
def checkandincrease(photoid):
    if ReportedPhotos.objects.filter(photoid=photoid).count() != 0:
        report = ReportedPhotos.objects.get(photoid=photoid)
    else:
        report = ReportedPhotos(photoid=photoid)

    report.howmany_reports += 1
    report.save()
    return True


# coding=utf-8
import requests
import codecs
from lxml import html
from lxml import etree
import  os,re
import time
import pymongo
import datetime
print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))

# http://blog.csdn.net/selectdb/article/details/14523205

def get_unix_time(s=u""):
    if u"天" in s:
        day = s.split(u'天')[0]
        return int(time.mktime(datetime.date(datetime.date.today().year, datetime.date.today().month, datetime.date.today().day-int(day)).timetuple()))
    elif u"月" in s:
        month = s.split(u"个")[0]
        return int(time.mktime(datetime.date(datetime.date.today().year, datetime.date.today().month-int(month), datetime.date.today().day).timetuple()))
    elif u"年" in s:
        year = s.split(u'年')[0]
        return int(time.mktime(datetime.date(datetime.date.today().year-int(year), datetime.date.today().month,  datetime.date.today().day).timetuple()))

def get_unix_time1(s=u""):
    # print(s)
    current_time = time.strftime('%Y-%m-%d', time.localtime(time.time()))
    day = s.split(' ')[0]
    t1 = day.split('/')
    f1 = 0
    for i in t1:
        f1 +=1
    hour = s.split(' ')[1]
    if u"今天" == day:
        s = current_time.replace('-', '/')+' '+hour+':'+'00'
    elif u"昨天" == day:
        t = current_time.split('-')
        current_time = str(t[0])+'/'+str(t[1])+'/'+str(int(t[2])-1)
        s = current_time+' '+hour + ':00'
    elif u"前天" == day:
        t = current_time.split('-')
        current_time = str(t[0])+'/'+str(t[1])+'/'+str(int(t[2])-2)
        s = current_time + ' ' + hour + ':00'
    elif f1 == 2:
        t = current_time.split('-')
        s = str(t[0]) + '/' + day+' '+hour+':00'
    else:
        s = s + ':'+'00'
    # print(s)
    return int(time.mktime(time.strptime(s, '%Y/%m/%d %H:%M:%S')))

def get1(url=''):
    try:
        conn = pymongo.MongoClient('localhost', 27017)
        db = conn.test
    except Exception as e:
        print("连接数据库失败！")
        return
    headers = {
    'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Accept-Encoding':'gzip, deflate, sdch, br',
    'Accept-Language':'zh-CN,zh;q=0.8',
    'Cache-Control':'max-age=0',
    'Connection':'keep-alive',
    'Cookie':'_user_behavior_=0421f753-481e-4765-ba5d-1724ab418db1; oscid=oFK7%2Bn8AFTbDIwoUMx0%2FjqaizEcjLGGK0sSawThXO5G%2BDoLLtdmsTIO8jDG5XBd3lEvcR7HRr5qypzmd30PKg8Y523vRM2i5EzJnLPnDasZEFz%2BEM%2F07Iz87%2BKOafyqgx7zSa%2FEjwNUNtb8QeK6VkA%3D%3D; Hm_lvt_a411c4d1664dd70048ee98afe7b28f0b=1478488050,1478512945,1478513960,1478566809; Hm_lpvt_a411c4d1664dd70048ee98afe7b28f0b=1478566809',
    'Host':'my.oschina.net',
    'Referer':'https://my.oschina.net/u/136849/home',
    'Upgrade-Insecure-Requests':'1',
    'User-Agent':'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/53.0.2785.143 Chrome/53.0.2785.143 Safari/537.36'
        }
    headers2 = {
    'Accept':'text/html,*/*;q = 0.01',
    'Accept-Encoding':'gzip, deflate, sdch, br',
    'Accept-Language':'zh - CN, zh;q = 0.8',
    'Connection':'keep-alive',
    'Content-Type':'application/x-www-form-urlencoded;charset=UTF-8',
    'Cookie':'_user_behavior_ = 0421f753-481e-4765-ba5d-1724ab418db1;oscid=oFK7%2Bn8AFTbDIwoUMx0%2FjqaizEcjLGGK0sSawThXO5G%2BDoLLtdmsTIO8jDG5XBd3lEvcR7HRr5qypzmd30PKg8Y523vRM2i5EzJnLPnDasZEFz%2BEM%2F07Iz87%2BKOafyqgx7zSa%2FEjwNUNtb8QeK6VkA%3D%3D;Hm_lvt_a411c4d1664dd70048ee98afe7b28f0b=1478488050,1478512945,1478513960,1478566809',
    'Host':'my.oschina.net',
    'Referer':'https://my.oschina.net/u/136848/home?type=bbs&scope=best',
    'User-Agent':'Mozilla/5.0(X11;Linuxi686) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/53.0.2785.143 Chrome/53.0.2785.143 Safari/537.36',
    'X-PJAX':'true',
    'X-PJAX-Container':'# main-article',
    'X-Requested-With':'XMLHttpRequest'
    }
    # 解析出错
    b = 0
    # 访问出错
    c = 0
    gender = ""
    avatar = ""
    username = ""
    company = ""
    location = ""
    position = ""
    point = 0
    follower = 0
    following = 0
    join_date = []
    rec_login_time ={}
    week_visits = 0
    month_visits = 0
    all_visits = 0
    # 个性签名
    signature = ""
    # 熟悉的开发平台
    fam_devplatform = []
    # 专长领域
    expertise = []
    # 软件作品
    software = []
    # 博文阅读数
    blog_readcount = []
    # 最佳答案
    best_answer = []
    for i in range(1, 2):
        # 访问网页异常处理
        try:
            s = requests.Session()
            # r = s.get('https://my.oschina.net/u/{0}/home?type=bbs&scope=best'.format(str(i)), headers=headers)
            r = s.get(url+'?type=bbs&scope=best', headers=headers)
            tree = html.fromstring(r.content.decode('utf-8'))
            # 解析网页异常处理
            try:
                # 元素是否存在异常处理
                try:
                    avatar = tree.xpath('//*[@class="user-icon"]/a/img/@src')[0]
                except Exception as e:
                    avatar = ""
                try:
                    username = tree.xpath('//*[@class="user-info-detail"]/div/a/span/text()')[0]
                except Exception as e:
                    username = ""
                try:
                    temp = tree.xpath('//*[@class="user-info-detail"]/div[1]/span[1]/text()')[0]
                    s1 = str(temp)
                    if('-' in s1):
                        company = s1.split('-')[0]
                        position = s1[s1.find('-', 0) + 2:len(s1)]
                    else:
                        company = ""
                        position = s1
                        # print(position)
                except Exception as e:
                    company = ""
                    position = ""
                try:
                    location = tree.xpath('//*[@class="user-info-detail"]/div[1]/span[2]/text()')[0]
                except Exception as e:
                    location = ""
                # 性别
                try:
                    gender = tree.xpath('/html/body/section/section[1]/div/div[1]/span/@class')[0]
                    gender = str(gender).split(' ')[1]
                except Exception as e:
                    gender = ""
                    print(e)
                # 加入时间
                join_datet = tree.xpath('//*[@id="user-score-pjax"]/div/div[4]/text()')[0]
                t4 = str(join_datet).strip().split(' ')
                join_datet = t4[1]
                tj = int(time.mktime(time.strptime(join_datet, '%Y/%m/%d')))
                tmpj = {"join_date":join_datet, "unix_timestamp": tj}
                join_date.append(tmpj)
                # 最近登录时间
                rec_login_time_t = tree.xpath('//*[@id="user-score-pjax"]/div/div[4]/text()')[0]
                rec_login_time_t = str(rec_login_time_t).strip()
                rec_login_time_t = rec_login_time_t[rec_login_time_t.find(u'录')+2:len(rec_login_time_t)]
                if rec_login_time_t != "":
                    unix_timestamp = get_unix_time1(rec_login_time_t)
                    rec_login_time = {"recent_login_time": rec_login_time_t, "unix_timestamp": unix_timestamp}
                else:
                    rec_login_time = {}

                # 积分
                point = int(tree.xpath('//*[@id="user-score-pjax"]/div/div[1]/a/text()')[0])
                # 粉丝
                follower = int(tree.xpath('//*[@id="user-score-pjax"]/div/div[2]/a/text()')[0])
                # 关注
                following = int(tree.xpath('//*[@id="user-score-pjax"]/div/div[3]/a/text()')[0])
                # 个性签名
                signature = tree.xpath('//*[@class="user-signature"]/span/text()')[0]
                if u'没写' in str(signature):
                    signature = ""
                # 本周访问
                week_visitst = tree.xpath('//*[@class="count-list"]/li[3]/text()')[0]
                week_visits = int(str(week_visitst)[str(week_visitst).find(':')+2:len(str(week_visitst))])
                # 本月访问
                month_visitst = tree.xpath('//*[@class="count-list"]/li[4]/text()')[0]
                month_visits = int(str(month_visitst)[str(month_visitst).find(':') + 2:len(str(month_visitst))])
                # 所有访问
                all_visitst = tree.xpath('//*[@class="count-list"]/li[5]/text()')[0]
                all_visits = int(str(all_visitst)[str(all_visitst).find(':') + 2:len(str(all_visitst))])
                # 熟悉平台
                try:
                    temp = tree.xpath('//div[@class="skills-item"]/p')[0]
                    t2 = 1
                    for k1 in temp.xpath('./span/text()'):
                        if(t2<5):
                            fam_devplatform.append(str(k1))
                            #print (fam_devplatform)
                            t2=t2+1
                        else:
                            break
                    #fam_devplatform=temp
                except Exception as e:
                    fam_devplatform = []

                # 专长领域
                try:
                    temp = tree.xpath('//div[@class="skills-item"]/p')[1]
                    t2=1
                    for k1 in temp.xpath('./span/text()'):
                        if(t2<=3):
                            expertise.append(str(k1))
                            t2=t2+1
                        else:
                            break
                except Exception as e:
                    expertise = []
                # 软件作品
                for ts in range(1, 10):
                    try:
                        s_title = tree.xpath('//*[@class="software-list"]/div[{0}]/div/a/div[1]/text()'.format(str(ts)))[0]
                        s_describe = tree.xpath('//*[@class="software-list"]/div[{0}]/div/a/div[2]/text()'.format(str(ts)))[0]
                        s_url = tree.xpath('//*[@class="software-list"]/div[{0}]/div/a/@href'.format(str(ts)))[0]
                        tmps = {"soft_title": s_title, "soft_describe": s_describe, "soft_url": s_url}
                        software.append(tmps)
                    except Exception as e:
                        break
                # 最佳回答
                for ta in range(1, 4):
                    try:
                        question = tree.xpath('//*[@id="main-anchor"]/div[2]/div[2]/div[{0}]/div[2]/div[2]/div[1]/a/@alt'.format(str(ta)))[0]
                        reply = int(tree.xpath('//*[@id="main-anchor"]/div[2]/div[2]/div[{0}]/div[2]/div[3]/span/span/text()'.format(str(ta)))[0])
                        rep_time = tree.xpath('//*[@id="main-anchor"]/div[2]/div/div[{0}]/div[2]/div[1]/span[2]/text()'.format(str(ta)))[0]
                        unix_timestamp = get_unix_time(rep_time)
                        tempa = {"question": question, "reply": reply, "reply_time": rep_time, "unix_timestamp": unix_timestamp}
                        best_answer.append(tempa)
                    except Exception as e:
                        break
            except Exception as e:
                print(e)
                print("index1")
                b= b+1

            # 博客阅读数
            try:
                s = requests.Session()
                url1 = url.split('home')[0]
                s.get(url1+'/blog/?sort=hot', headers=headers2)
                # r = s.get('https://my.oschina.net/u/136848?sort=hot', headers=headers3)
                tree = html.fromstring(r.content.decode('utf-8'))
                # print(r.content.decode('utf-8'))
                for tm in range(1, 6):
                    try:
                        title = tree.xpath('//*[@id="list"]/div[{0}]/div/div/div[1]/a/text()'.format(str(tm)))[0]
                        read_num = int(tree.xpath('//*[@id="list"]/div[{0}]/div/div/div[2]/ul//li[1]/span/text()'.format(str(tm)))[0])
                        comment_num = int(tree.xpath('//*[@id="list"]/div[{0}]/div/div/div[2]/ul//li[2]/span/text()'.format(str(tm)))[0])
                        com_time = tree.xpath('//*[@id="list"]/div[{0}]/div/div/div[3]/text()'.format(str(tm)))[0]
                        com_time = str(com_time).strip().split(' ')[0]
                        unix_timestamp = int(time.mktime(time.strptime(com_time, '%Y/%m/%d')))
                        tempm = {"title": str(title).strip(), "read_count": read_num, "comment_count": comment_num,
                                 "comment_time": com_time, "unix_timestamp": unix_timestamp}
                        blog_readcount.append(tempm)
                    except Exception as e:
                        break
            except Exception as e:
                print(e)
                print("index4")
                b=b+1
            db.osChinaTest1.insert({"avatar": avatar,
                                   "gender": gender,
                                   "username": username,
                                   "company": company,
                                   "location": location,
                                   "position": position,
                                   "point": point,
                                   "follower": follower,
                                   "following": following,
                                   "join_date": join_date,
                                   "recent_login_time": rec_login_time,
                                   "week_visits": week_visits,
                                   "month_visits": month_visits,
                                   "all_visits": all_visits,
                                   "signature": signature,
                                   "familiar_develop_platform": fam_devplatform,
                                   "expertise": expertise,
                                   "software": software,
                                   "best_answer": best_answer,
                                   "blog_read_count": blog_readcount
                                   })
            fam_devplatform.clear()
            expertise.clear()
            software.clear()
            best_answer.clear()
            blog_readcount.clear()
            join_date.clear()
            rec_login_time.clear()
        except Exception as e:
            c = c+1
            print(e)
    # print("解析网页失败:"+str(b))
    # print("访问网页失败:"+str(c))

con = pymongo.MongoClient('localhost', 27017)
d = con.test
cu = d.Url
r = cu.find({})
t3 = 0
for i in r:
    t3 += 1
    url = i["url"]
    get1(url)
    if t3 == 2:
        break
print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))


# -*- coding:utf-8 -*-
import os
import re
import sys
import json
import zmq
import errno
import signal
from functools import wraps
from ee.common import xavier

handle_errorno = {
    #校验不过(error: the CRC)
    'handle_errno_crc_checksum_failure':-67, \
    #无权限(error: the instruction need permissions)
    'handle_errno_no_permission':-68, \
    #/任务正在运行(error: the instruction also is doing.which need long time)
    'handle_errno_running':-69, \
    'handle_errno_devices_error':-70, \
    'handle_errno_time_out':-75, \
    'handle_errno_call_handle_exception':-76, \
    'handle_errno_not_have_this_command':-190,\
    #参数出错-193...-223：参数\参数1...参数30错误
    'handle_errno_parameter_invalid':-193, \
    #执行出错-224...-254：执行\执行1...执行30错误(error: casued by ARM board doing)
    'handle_errno_execute_failure':-224,
    }

def handle_done(backstring = ""):
    sResult = "Done"
    sResult += backstring
    return sResult

def handle_error(errorno,backstring = ""):
    sResult = "Error"
    sResult += str(errorno)
    sResult += " "
    sResult += backstring
    return sResult

def is_ask_for_help(params):
    if len(params) == 1 and params[0] == "?":
        return True
    else :
        return False

class TimeoutError(Exception):
    pass

def timeout(seconds=5, error_message=os.strerror(errno.ETIME)):
    def decorator(func):
        def _handle_timeout(signum, frame):
            raise TimeoutError(error_message)

        def wrapper(*args, **kwargs):
            signal.signal(signal.SIGALRM, _handle_timeout)
            signal.alarm(seconds)
            try:
                result = func(*args, **kwargs)
            except TimeoutError:
                try:
                    xavier.proxy.reconnect()
                except Exception:
                    pass
                result = handle_error(handle_errorno["handle_errno_time_out"], "call handle timeout")
            finally:
                signal.alarm(0)
            return result

        return wraps(func)(wrapper)

    return decorator