import numpy as np
import pandas as pd
import pandas_datareader as web
from datetime import date
from dateutil.relativedelta import relativedelta
import scipy.optimize as optimize
import plotly
import plotly.graph_objects as go
import json

class EF:

    def __init__(self, duration, pl_data, pl_tickers, pl_weights, num_ports):
        self.pl_data = pl_data
        self.tickers = pl_tickers
        self.weights = pl_weights
        self.end = date.today()
        self.start = self.end - relativedelta(years=duration)
        self.num_ports = num_ports
        self.num_portfolios = 100
        self.risk_free_rate = 0.0152

    def get_data(self):
        data = {}
        for ticker in self.tickers:
            data[ticker] = web.DataReader(ticker, 'yahoo',
                                          self.start.strftime('%Y-%m-%d'),
                                          self.end.strftime('%Y-%m-%d'))[
                'Adj Close']
        df = pd.DataFrame.from_dict(data)
        return df

    def get_return(self):
        df = self.get_data()
        returns = df.pct_change()
        mean_returns = returns.mean()
        cov_matrix = returns.cov()

        return returns, mean_returns, cov_matrix

    def my_current_portfolio(self):
        df = self.get_data()
        # Re-balance - normalization
        weights = np.array(self.weights / np.sum(self.weights))

        ## Expected Return
        exp_ret = np.sum(df.pct_change().mean() * weights) * 252

        ## Expected Volatility
        exp_vol = np.sqrt(
            np.dot(weights.T, np.dot(df.pct_change().cov() * 252, weights)))

        ## Sharpe Ratio
        SR = exp_ret / exp_vol

        output = ["Current Portfolio", round(exp_ret * 100, 2),
                  round(exp_vol * 100, 2), round(SR, 4)]
        return output

    def generate_ef_viz(self):
        my_pl_output = self.my_current_portfolio()
        df = self.get_data()
        returns, mean_returns, cov_matrix = self.get_return()
        annual_volatility, annual_return, expected_volatility, expected_return, \
        min_expected_volatility, min_expected_return, max_sharpe, min_vol, \
        frontier_vol, frontier_y, ef_df, \
        max_sharpe_allocation, min_vol_allocation = self.display_ef(
            mean_returns,
            cov_matrix,
            self.risk_free_rate,
            self.num_portfolios,
            returns, df,
            self.tickers)

        tbl_contents = \
            [list(e) for e in zip(*[my_pl_output,
                                    [
                                        "Maximum Sharpe Ratio Portfolio",
                                        round(expected_return * 100,
                                              2),
                                        round(
                                            expected_volatility * 100,
                                            2),
                                        round(
                                            expected_return / expected_volatility,
                                            4)],
                                    ["Minimum Volatility Portfolio",
                                     round(
                                         min_expected_return * 100,
                                         2),
                                     round(
                                         min_expected_volatility * 100,
                                         2),
                                     round(
                                         min_expected_return / min_expected_volatility,
                                         4)],
                                    ])]

        # PLOT 1 PORTFOLIO COMPARISON =========================================
        tbl_fig = go.Figure()
        tbl_fig.add_trace(go.Table(
            header=dict(
                values=["Portfolio", "Expected Return", "Expected Volatility",
                        "Sharpe Ratio"],
                font=dict(size=15),
                align="left"
            ),
            cells=dict(
                values=tbl_contents,
                align="left")
        ))
        tbl_fig.update_layout(
            title={'text': "Portfolio Comparison"},
            titlefont=dict(size=36),
            height=250
        )

        TBLgraphJSON = json.dumps(tbl_fig, cls=plotly.utils.PlotlyJSONEncoder)

        # PLOT 2 ALLOCATION TABLE ===========================================
        weights_contents = \
            [list(e) for e in zip(*[
                ["Maximum Sharpe Ratio Portfolio"] + max_sharpe.x.tolist(),
                ["Minimum Volatility Portfolio"] + min_vol.x.tolist()
                                    ])]
        alloc_fig = go.Figure()
        alloc_fig.add_trace(go.Table(
            header=dict(
                values=['Portfolio'] + max_sharpe_allocation.columns.to_list(),
                font=dict(size=15),
                align="left"
            ),
            cells=dict(
                values=weights_contents,
                align="left")
        ))
        alloc_fig.update_layout(
            title={'text': "Allocation Weights"},
            titlefont=dict(size=36),
            height=250
        )

        ALLOCgraphJSON = json.dumps(alloc_fig, cls=plotly.utils.PlotlyJSONEncoder)

        # PLOT 3 EF PLOT ======================================================
        vol_arr, ret_arr, sharpe_arr = self.run_simulation(df)
        ef_fig = go.Figure()

        # Mark the tickers position
        ef_fig.add_trace(go.Scatter(
            x=annual_volatility * 100,
            y=annual_return * 100,
            mode="markers",
            marker=dict(
                size=16,
                symbol='diamond',
                color='orange'
            ),
            name="Ticker",
            text=self.tickers,
            # TODO check if this worked https://plotly.com/python/text-and-annotations/
            # textposition="bottom center",
            hovertemplate=
            'Ticker: %{text}' +
            '
Expected Return: %{y:.2f}%
' +
            'Risk: %{x:.2f}%'
        ))

        # Simulation
        ef_fig.add_trace(go.Scatter(
            x=vol_arr * 100,
            y=ret_arr * 100,
            mode="markers",
            marker=dict(
                color=sharpe_arr,
            ),
            name="EF Simulation",
            text=sharpe_arr,
            hovertemplate=
            'Simulated Allocation' +
            '
Sharpe Ratio: %{text:.2f}' +
            '
Expected Return: %{y:.2f}%
' +
            'Risk: %{x:.2f}%'

        ))

        # Frontier line
        ef_fig.add_trace(go.Scatter(
            x=ef_df['Standard Deviation'] * 100,
            y=ef_df['Expected Return'] * 100,
            mode='lines+markers',
            line=dict(color="black"),
            name="EF Line",
            text=ef_df['Sharpe Ratio'],
            hovertemplate=
            'Efficient Frontier Line' +
            '
Sharpe Ratio: %{text:.2f}' +
            '
Expected Return: %{y:.2f}%
' +
            'Risk: %{x:.2f}%'
        ))
        # Maximum Sharpe Ratio Portfolio Allocation
        ef_fig.add_trace(go.Scatter(
            x=[expected_volatility * 100],
            y=[expected_return * 100],
            mode="markers",
            name="Max. SR. ALLOC.",
            marker=dict(
                size=16,
                symbol='star',
                color='green'
            ),
            text=[expected_return / expected_volatility],
            hovertemplate=
            'Maximum Sharpe Ratio Portfolio ALLOC' +
            '
Sharpe Ratio: %{text:.2f}' +
            '
Expected Return: %{y:.2f}%
' +
            'Risk: %{x:.2f}%'
        ))

        # Minimum Volatility Portfolio Allocation
        ef_fig.add_trace(go.Scatter(
            x=[min_expected_volatility * 100],
            y=[min_expected_return * 100],
            mode="markers",
            name="Min. Risk ALLOC.",
            marker=dict(
                size=16,
                symbol='x-dot',
                color='blue'
            ),
            text=[min_expected_volatility / min_expected_return],
            hovertemplate=
            'Minimum Volatility Portfolio ALLOC' +
            '
Sharpe Ratio: %{text:.2f}' +
            '
Expected Return: %{y:.2f}%
' +
            'Risk: %{x:.2f}%'
        ))
        tbl_fig.update_layout(
            title="Portfolio Comparison",

        )
        ef_fig.update_layout(
            xaxis_title="Standard Deviation",
            yaxis_title="Expected Return",
            title="Efficient Frontier",
            titlefont=dict(size=36),
            # legend_title='Tickers',
            showlegend=True,
            height=700,

            xaxis=dict(
                # type='linear',
                ticksuffix='%'),
            yaxis=dict(
                # type='linear',
                # range=[1, 100],
                ticksuffix='%'),
            # hovermode="x"
        )

        EFgraphJSON = json.dumps(ef_fig, cls=plotly.utils.PlotlyJSONEncoder)

        # PLOT 4 Random Allocation Simulation =================================
        traces = {}

        for col in self.tickers:
            traces[col] = go.Scatter(x=ef_df['Standard Deviation'] * 100,
                                     y=ef_df[col] * 100,
                                     name=col,
                                     # text=[round(c,2) for c in col],
                                     # mode='lines',
                                     # line=dict(width=0.5),
                                     stackgroup='one'
                                     )
        sim_data = list(traces.values())
        sim_fig = go.Figure(sim_data)
        sim_fig.update_traces(
            mode="lines",
            # hovertemplate=None
            hovertemplate=
            'Allocation: %{y:.2f}%'
            # 'Risk: %{x:.2f}%'
            # hovertemplate="%{y}"
        )
        sim_fig.update_layout(
            xaxis_title="Standard Deviation",
            yaxis_title="Allocation",
            title="Random Allocation Simulation",
            titlefont=dict(size=36),
            legend_title='Tickers',
            showlegend=True,
            xaxis=dict(
                type='linear',
                ticksuffix='%'),
            yaxis=dict(
                type='linear',
                range=[1, 100],
                ticksuffix='%'),
            hovermode="x")
        SIMgraphJSON = json.dumps(sim_fig, cls=plotly.utils.PlotlyJSONEncoder)

        return TBLgraphJSON, EFgraphJSON, SIMgraphJSON, ALLOCgraphJSON

    # bring EF.py

    def portfolio_annualised_performance(self, weights, mean_returns,
                                         cov_matrix):
        """
        :param weights:
        :param mean_returns:
        :param cov_matrix:
        :return: expected volatility, expected return
        """
        # Expected Return
        returns = np.sum(mean_returns * weights) * 252
        # Expected Volatility
        # Denominator of the sharpe ratio
        # Use Linear Algebra here
        # transpose the weights
        # dot product of log_returns covariance multiplied by 252 with weights
        # sqrt of whole thing gives the expected volatility
        std = np.sqrt(
            np.dot(weights.T, np.dot(cov_matrix, weights))) * np.sqrt(
            252)
        return std, returns

    def random_portfolios(self, num_portfolios, mean_returns, cov_matrix,
                          risk_free_rate, tickers):
        # results for vol, return, sharpe ratio
        results = np.zeros((3, num_portfolios))
        # allocation ratio
        weights_record = []
        for i in range(num_portfolios):
            # create random weights for number of tickers
            weights = np.random.random(len(tickers))
            # rebalance - normalization
            weights = weights / np.sum(weights)
            weights_record.append(weights)
            portfolio_std_dev, portfolio_return = self.portfolio_annualised_performance(
                weights, mean_returns, cov_matrix)
            # Standard Deviation
            results[0, i] = portfolio_std_dev
            # Expected Return
            results[1, i] = portfolio_return
            # Sharpe Ratio
            results[2, i] = (
                                    portfolio_return - risk_free_rate) / portfolio_std_dev

        weights_df = pd.DataFrame.from_records(weights_record, columns=tickers)
        result_df = pd.DataFrame.from_records(results.T,
                                              columns=['Standard Deviation',
                                                       'Expected Return',
                                                       'Sharpe Ratio'])
        portfolio_allocation_df = pd.concat([weights_df, result_df],
                                            axis=1).sort_values(
            by=['Expected Return'], ascending=False)
        return portfolio_allocation_df

    def neg_sharpe_ratio(self, weights, mean_returns, cov_matrix,
                         risk_free_rate):
        p_var, p_ret = self.portfolio_annualised_performance(weights,
                                                             mean_returns,
                                                             cov_matrix)
        return -(p_ret - risk_free_rate) / p_var

    def max_sharpe_ratio(self, mean_returns, cov_matrix, risk_free_rate):
        num_assets = len(mean_returns)
        args = (mean_returns, cov_matrix, risk_free_rate)
        constraints = ({'type': 'eq', 'fun': lambda x: np.sum(x) - 1})
        bound = (0.0, 1.0)
        bounds = tuple(bound for asset in range(num_assets))
        result = optimize.minimize(self.neg_sharpe_ratio,
                                   num_assets * [1. / num_assets, ],
                                   args=args,
                                   method='SLSQP', bounds=bounds,
                                   constraints=constraints,
                                   options={"disp": True})
        return result

    def portfolio_volatility(self, weights, mean_returns, cov_matrix):
        return self.portfolio_annualised_performance(weights, mean_returns,
                                                     cov_matrix)[
            0]

    def min_variance(self, mean_returns, cov_matrix):
        num_assets = len(mean_returns)
        args = (mean_returns, cov_matrix)
        constraints = ({'type': 'eq', 'fun': lambda x: np.sum(x) - 1})
        bound = (0.0, 1.0)
        bounds = tuple(bound for asset in range(num_assets))

        result = optimize.minimize(self.portfolio_volatility,
                                   num_assets * [1. / num_assets, ], args=args,
                                   method='SLSQP', bounds=bounds,
                                   constraints=constraints)

        return result

    def efficient_return(self, mean_returns, cov_matrix, target):
        num_assets = len(mean_returns)
        args = (mean_returns, cov_matrix)

        def portfolio_return(weights):
            return self.portfolio_annualised_performance(weights, mean_returns,
                                                         cov_matrix)[1]

        constraints = (
            {'type': 'eq', 'fun': lambda x: portfolio_return(x) - target},
            {'type': 'eq', 'fun': lambda x: np.sum(x) - 1})
        bounds = tuple((0, 1) for asset in range(num_assets))
        result = optimize.minimize(self.portfolio_volatility,
                                   num_assets * [1. / num_assets, ], args=args,
                                   method='SLSQP', bounds=bounds,
                                   constraints=constraints)
        return result

    def efficient_frontier(self, mean_returns, cov_matrix, returns_range):
        efficients = []
        for ret in returns_range:
            efficients.append(
                self.efficient_return(mean_returns, cov_matrix, ret))
        return efficients

    def display_ef(self, mean_returns, cov_matrix, risk_free_rate,
                   num_portfolios,
                   returns, df, tickers):
        port_alloc_df = self.random_portfolios(num_portfolios, mean_returns,
                                               cov_matrix,
                                               risk_free_rate, tickers)

        max_sharpe = self.max_sharpe_ratio(mean_returns, cov_matrix,
                                           risk_free_rate)
        # expected volatility, expected return
        expected_volatility, expected_return = self.portfolio_annualised_performance(
            max_sharpe['x'],
            mean_returns,
            cov_matrix)  # weights,
        max_sharpe_allocation = pd.DataFrame(max_sharpe.x, index=df.columns,
                                             columns=['allocation'])
        max_sharpe_allocation.allocation = [round(i * 100, 2) for i in
                                            max_sharpe_allocation.allocation]
        max_sharpe_allocation = max_sharpe_allocation.T

        # Minimum volatility, minimum expected return
        min_vol = self.min_variance(mean_returns, cov_matrix)
        min_expected_volatility, min_expected_return = self.portfolio_annualised_performance(
            min_vol['x'],
            mean_returns,
            cov_matrix)
        min_vol_allocation = pd.DataFrame(min_vol.x, index=df.columns,
                                          columns=['allocation'])
        min_vol_allocation.allocation = [round(i * 100, 2) for i in
                                         min_vol_allocation.allocation]
        min_vol_allocation = min_vol_allocation.T

        annual_volatility = np.std(returns) * np.sqrt(252)
        annual_return = mean_returns * 252

        print("-" * 80)
        print("Maximum Sharpe Ratio Portfolio Allocation\n")
        print("Annualised Return:", expected_return)
        print("Annualised Volatility:", expected_volatility)
        print(max_sharpe_allocation)  # red star
        print("-" * 80)

        print("Minimum Volatility Portfolio Allocation\n")
        print("Annualised Return:", min_expected_return)
        print("Annualised Volatility:", min_expected_volatility)
        print(min_vol_allocation)  # green star
        print("-" * 80)
        print("Individual Stock Returns and Volatility\n")
        for i, txt in enumerate(df.columns):
            print(txt, ":", "annualised return", annual_return[i],
                  "| annualised volatility:", annual_volatility[i])
        print("-" * 80)

        frontier_y = np.linspace(min_expected_return, expected_return,
                                 100)  # This is frontier_y, expected return
        efficient_portfolios = self.efficient_frontier(mean_returns,
                                                       cov_matrix,
                                                       frontier_y)

        # fun is Volatility, x is Allocation, frontier_y=target is expected Return / sharpe ratio? ret / vol

        frontier_vol = [p['fun'] for p in efficient_portfolios]
        ef_alloc = [p['x'] for p in efficient_portfolios]
        ef_sr = frontier_y / np.array(frontier_vol)

        ef_alloc_df = pd.DataFrame.from_records(ef_alloc,
                                                columns=tickers)  # 10,000
        ef_vol_df = pd.DataFrame(frontier_vol,
                                 columns=['Standard Deviation'])  # 10,000
        ef_ret_df = pd.DataFrame(frontier_y,
                                 columns=['Expected Return'])  # 100
        ef_sr_df = pd.DataFrame(ef_sr, columns=['Sharpe Ratio'])  # 100

        ef_df = pd.concat([ef_alloc_df, ef_vol_df, ef_sr_df, ef_ret_df],
                          axis=1).sort_values(by=['Expected Return'])
        return annual_volatility, annual_return, expected_volatility, \
               expected_return, min_expected_volatility, min_expected_return, \
               max_sharpe, min_vol, frontier_vol, frontier_y, ef_df, \
               max_sharpe_allocation, min_vol_allocation

    def run_simulation(self, df):
        all_weights = np.zeros((self.num_portfolios, len(df.columns)))
        ret_arr = np.zeros(self.num_portfolios)
        vol_arr = np.zeros(self.num_portfolios)
        sharpe_arr = np.zeros(self.num_portfolios)

        for ind in range(self.num_portfolios):
            # Create Random Weights
            weights = np.array(np.random.random(len(self.tickers)))

            # Rebalance Weights
            weights = weights / np.sum(weights)

            # Save Weights
            all_weights[ind, :] = weights

            # Expected Return
            ret_arr[ind] = np.sum((df.pct_change().mean() * weights) * 252)

            # Expected Variance
            vol_arr[ind] = np.sqrt(
                np.dot(weights.T,
                       np.dot(df.pct_change().cov() * 252, weights)))

            # Sharpe Ratio
            sharpe_arr[ind] = ret_arr[ind] / vol_arr[ind]

        return vol_arr, ret_arr, sharpe_arr


import concurrent.futures
import logging
import os
from datetime import datetime, timedelta
from string import Template

import boto3
import botocore
from airflow import DAG
from airflow.models import Variable
from airflow.operators.python_operator import PythonOperator
from mys3utils.tools import FETCHES_BUCKET

from localutils import get_file_list, get_prefix_from_template as get_prefix

def generate_object_list(*args, **kwargs):
    prefix = get_prefix(**kwargs)
    logging.info(f'Will be getting objects for {prefix}')
    pfl = get_file_list(prefix=prefix, **kwargs)
    pfl.update()
    pfl.store()

def download_and_store(**kwargs):
    prefix = get_prefix(**kwargs)
    target_dir = os.path.join(Variable.get('target_dir'), prefix)
    os.makedirs(target_dir, exist_ok=True)

    pfl = get_file_list(prefix=prefix, **kwargs)
    pfl.load()
    logging.info(f"Downloading {len(pfl.get_list())} objects from {prefix} to {target_dir}")

    client = boto3.client('s3', config=botocore.client.Config(
        signature_version=botocore.UNSIGNED))

    def myfunc(obj, client=client):
        if obj['Name'].endswith('/'):
            return 'skipped'
        local_name = os.path.join(target_dir, obj['Name'].split('/')[-1])
        client.download_file(Bucket=FETCHES_BUCKET,
                             Key=obj['Name'], Filename=local_name)
        return 'Done'

    with concurrent.futures.ThreadPoolExecutor() as executor:
        for obj, status in zip(pfl.get_list(), executor.map(myfunc, pfl.get_list())):
            logging.info(f"{ obj['Name']} status: { status }")

default_args = {
    'owner': 'airflow',
    'depends_on_past': False,
    'start_date': datetime(2013, 11, 26),
    'end_date': datetime(2013, 12, 28),
    'provide_context': True,
    'catchup': True
}

op_kwargs = {
    #    'prefix-pattern': 'test-realtime-gzip/$date/',
    'base_dir': '/tmp/'
}

dag = DAG('downloader', default_args=default_args,
          schedule_interval=timedelta(1))

get_objects_task = PythonOperator(task_id='get_object_list',
                                  python_callable=generate_object_list,
                                  op_kwargs=op_kwargs,
                                  dag=dag)

download_task = PythonOperator(task_id='download',
                               python_callable=download_and_store,
                               op_kwargs=op_kwargs,
                               dag=dag)

get_objects_task >> download_task


# -*- coding: utf-8 -*-
#
# Copyright © Spyder Project Contributors
# Licensed under the terms of the MIT License

"""Tests for jedi_plugin.py"""

from textwrap import dedent

import pytest

from spyder.utils.introspection.manager import CodeInfo
from spyder.utils.introspection import jedi_plugin

try:
    import numpydoc
except ImportError:
    numpydoc = None

try:
    import numpy
except ImportError:
    numpy = None

try:
    import matplotlib
except ImportError:
    matplotlib = None

p = jedi_plugin.JediPlugin()
p.load_plugin()

def test_get_info():
    source_code = "import os; os.walk"
    docs = p.get_info(CodeInfo('info', source_code, len(source_code)))
    assert docs['calltip'].startswith('walk(') and docs['name'] == 'walk'

def test_get_completions():
    source_code = "import o"
    completions = p.get_completions(CodeInfo('completions', source_code,
                                             len(source_code)))
    assert ('os', 'module') in completions

def test_get_definition():
    source_code = "import os; os.walk"
    path, line_nr = p.get_definition(CodeInfo('definition', source_code,
                                              len(source_code)))
    assert 'os.py' in path

def test_get_path():
    source_code = 'from spyder.utils.introspection.manager import CodeInfo'
    path, line_nr = p.get_definition(CodeInfo('definition', source_code,
                                              len(source_code), __file__))
    assert 'utils' in path and 'introspection' in path

def test_get_docstring():
    source_code = dedent('''
    def test(a, b):
        """Test docstring"""
        pass
    test''')
    path, line = p.get_definition(CodeInfo('definition', source_code,
                                           len(source_code), 'dummy.txt',
                                           is_python_like=True))
    assert line == 2

    docs = p.get_info(CodeInfo('info', source_code, len(source_code),
                               __file__))
    assert 'Test docstring' in docs['docstring']

@pytest.mark.skipif(not(numpy and numpydoc),
                    reason="numpy and numpydoc required")
def test_numpy_returns():
    source_code = dedent('''
    import numpy as np
    x = np.array([1,2,3])
    x.a''')
    completions = p.get_completions(CodeInfo('completions', source_code,
                                             len(source_code)))
    assert ('argmax', 'function') in completions

@pytest.mark.skipif(not(matplotlib and numpydoc),
                    reason="matplotlib required")
def test_matplotlib_fig_returns():
    source_code = dedent('''
    import matplotlib.pyplot as plt
    fig = plt.figure()
    fig.''')
    completions = p.get_completions(CodeInfo('completions', source_code,
                                             len(source_code)))
    assert ('add_axes', 'function') in completions

if __name__ == '__main__':
    pytest.main()


# Global / system
import os
import csv
import uuid
import asyncio

from pygazebo.msg import world_control_pb2, request_pb2, poses_stamped_pb2, gz_string_pb2
from sdfbuilder.math import Vector3

from .robot import Robot

from revolve.spec.msgs.model_inserted_pb2 import ModelInserted
from revolve.util import Time

from .connect import connect, RequestHandler
from ..logging import logger

class WorldManager(object):
    """
    A WorldManager utility class with methods more suited to
    Revolve.Angle, such as inserting whole robot trees etc.
    """

    def __init__(self, world_address=None, analyzer_address=None,
                 output_directory=None, pose_update_frequency=None,
                 restore=None):
        """

        :param restore: Restore the world from this directory, if available. Only works
                         if `output_directory` is also specified.
        :param pose_update_frequency:
        :param generator:
        :param _private:
        :param world_address:
        :param analyzer_address:
        :param builder:
        :param output_directory:
        :return:
        """

        self.world_address = world_address
        self.connection = None
        self.world_control = None
        self.unique_id = uuid.uuid4().time_mid

        # Output files for robot CSV data
        self.robots_file = None
        self.poses_file = None
        # self.write_robots = None
        # self.write_poses = None
        self.output_directory = None
        self.robots_filename = None
        self.poses_filename = None
        self.snapshot_filename = None
        self.world_snapshot_filename = None

        self.pose_update_frequency = pose_update_frequency

        self.robots = {}
        self.robot_id = 0

        self.start_time = None
        self.last_time = None

        # List of functions called when the local state updates
        self.update_triggers = []

        self.do_restore = None

        if output_directory:
            if not restore:
                restore = datetime.now().strftime(datetime.now().strftime('%Y%m%d%H%M%S'))

            self.output_directory = os.path.join(output_directory, restore)

            if not os.path.exists(self.output_directory):
                os.mkdir(self.output_directory)

            self.snapshot_filename = os.path.join(self.output_directory, 'snapshot.pickle')
            if os.path.exists(self.snapshot_filename):
                # Snapshot exists - restore from it
                with open(self.snapshot_filename, 'rb') as snapshot_file:
                    self.do_restore = pickle.load(snapshot_file)

            self.world_snapshot_filename = os.path.join(self.output_directory, 'snapshot.world')

            self.robots_filename = os.path.join(self.output_directory, 'robots.csv')
            self.poses_filename = os.path.join(self.output_directory, 'poses.csv')

            if self.do_restore:
                # Copy snapshot files and open created files in append mode
                # TODO Delete robot sdf / pb files that were created after the snapshot
                shutil.copy(self.poses_filename+'.snapshot', self.poses_filename)
                shutil.copy(self.robots_filename+'.snapshot', self.robots_filename)

                self.robots_file = open(self.robots_filename, 'ab')
                self.poses_file = open(self.poses_filename, 'ab')
                # self.write_robots = csv.writer(self.robots_file, delimiter=',')
                # self.write_poses = csv.writer(self.poses_file, delimiter=',')
            else:
                # Open poses file, this is written *a lot* so use default OS buffering
                self.poses_file = open(os.path.join(self.output_directory, 'poses.csv'), 'wb')

                # Open robots file line buffered so we can see it on the fly, isn't written
                # too often.
                self.robots_file = open(os.path.join(self.output_directory, 'robots.csv'), 'wb', buffering=1)
                # self.write_robots = csv.writer(self.robots_file, delimiter=',')
                # self.write_poses = csv.writer(self.poses_file, delimiter=',')

                # self.write_robots.writerow(self.robots_header())
                # self.write_poses.writerow(self.poses_header())

    # async def create(cls, world_address=("127.0.0.1", 11345), pose_update_frequency=10):
    #     self = cls(world_address=world_address, pose_update_frequency=pose_update_frequency)
    #     await self._init()
    #     return self

    async def _init(self):
        """
        Initializes connections for the world manager
        :return:
        """
        if self.connection is not None:
            return

        # Initialize the manager / analyzer connections as well as
        # the general request handler
        self.connection = await connect(self.world_address)

        self.world_control = await self.connection.advertise(
            '/gazebo/default/world_control', 'gazebo.msgs.WorldControl')

        # Wait for connections
        await self.world_control.wait_for_listener()

        # Subscribe to pose updates
        self.pose_subscriber = self.connection.subscribe(
            '/gazebo/default/revolve/robot_poses',
            'gazebo.msgs.PosesStamped',
            self._update_poses)

        # Wait for connections
        await self.pose_subscriber.wait_for_connection()

        self.request_handler = await RequestHandler.create(self.connection)

        await self.set_pose_update_frequency(self.pose_update_frequency)

        if self.do_restore:
            self.restore_snapshot(self.do_restore)

    async def pause(self, pause):
        """
        Pause / unpause the world
        :param pause:
        :return: Future for the published message
        """
        if pause:
            logger.debug("Pausing the world.")
        else:
            logger.debug("Unpausing the world.")

        msg = world_control_pb2.WorldControl()
        msg.pause = pause
        return await self.world_control.publish(msg)

    def insert_model(self, sdf):
        """
        Insert a model wrapped in an SDF tag into the world. Make
        sure it has a unique name, as it will be literally inserted into the world.

        This coroutine yields until the request has been successfully sent.
        It returns a future that resolves when a response has been received. The
        optional given callback is added to this future.

        :param sdf:
        :type sdf: SDF
        :return:
        """
        return self._do_gazebo_request("insert_sdf", data=str(sdf))

    def delete_model(self, name, req="entity_delete"):
        """
        Deletes the model with the given name from the world.
        :param name:
        :param req: Type of request to use. If you are going to
        delete a robot, I suggest using `delete_robot` rather than `entity_delete`
        because this attempts to prevent some issues with segmentation faults
        occurring from deleting sensors.
        :return:
        """
        return self._do_gazebo_request(req, data=name)

    def get_unique_id(self):
        self.unique_id += 1
        return self.unique_id

    def robots_header(self):
        """
        Returns the header to be written to the robots file
        :return:
        """
        return ['id', 'parent1', 'parent2']

    def poses_header(self):
        """
        Returns the header to be written to the poses file
        :return:
        """
        return ['id', 'sec', 'nsec', 'x', 'y', 'z']

    def teardown(self):
        """
        Finalizes the world, flushes files, etc.
        :return:
        """
        if self.robots_file:
            self.robots_file.close()
            self.poses_file.close()

    async def create_snapshot(self):
        """
        Creates a snapshot of the world in the output directory. This pauses the world.
        :return:
        """
        if not self.output_directory:
            logger.warning("No output directory - no snapshot will be created.")
            return False

        # Pause the world
        await self.pause(True)

        # Obtain a copy of the current world SDF from Gazebo and write it to file
        response = await self._do_gazebo_request("world_sdf")
        if response.response == "error":
            logger.warning("WARNING: requesting world state resulted in error. Snapshot failed.")
            return False

        msg = gz_string_pb2.GzString()
        msg.ParseFromString(response.serialized_data)
        with open(self.world_snapshot_filename, 'wb') as f:
            f.write(msg.data)

        # Get the snapshot data and pickle to file
        data = self.get_snapshot_data()

        with open(self.snapshot_filename, 'wb') as f:
            pickle.dump(data, f)

        # Flush statistic files and copy them
        self.poses_file.flush()
        self.robots_file.flush()
        shutil.copy(self.poses_filename, self.poses_filename+'.snapshot')
        shutil.copy(self.robots_filename, self.robots_filename+'.snapshot')
        return True

    def restore_snapshot(self, data):
        """
        Called with the data object created and pickled in `get_snapshot_data`,
        should restore the state of the world manager to where
        it can continue the way it left off.
        :param data:
        :return:
        """
        self.robots = data['robots']
        self.robot_id = data['robot_id']
        self.start_time = data['start_time']
        self.last_time = data['last_time']

    def get_snapshot_data(self):
        """
        Returns a data object to be pickled into a snapshot file.
        This should contain
        :return:
        """
        return {
            "robots": self.robots,
            "robot_id": self.robot_id,
            "start_time": self.start_time,
            "last_time": self.last_time
        }

    def _do_gazebo_request(self, request, data=None, dbl_data=None):
        """
        Convenience wrapper to use `do_request` with a default Gazebo
        `Request` message. See that method for more info.

        :param request:
        :type request: str
        :param data:
        :param dbl_data:
        :param msg_id: Force the message to use this ID. Sequencer is used if no message
                       ID is specified.
        :type msg_id: int
        :return:
        """
        req = request_pb2.Request()
        req.id = self.get_unique_id()
        req.request = request

        if data is not None:
            req.data = data

        if dbl_data is not None:
            req.dbl_data = dbl_data

        return self.request_handler.do_request(req)

    def set_pose_update_frequency(self, freq):
        """
        Sets the pose update frequency. Defaults to 10 times per second.
        :param freq:
        :type freq: int
        :return:
        """
        self.pose_update_frequency = freq
        return self._do_gazebo_request("set_robot_pose_update_frequency", str(freq))

    def get_robot_id(self):
        """
        Robot ID sequencer
        :return:
        """
        self.robot_id += 1
        return self.robot_id

    def robot_list(self):
        """
        Returns the list of registered robots
        :return:
        :rtype: list[Robot]
        """
        return self.robots.values()

    def get_robot_by_name(self, name):
        """
        :param name:
        :return:
        :rtype: Robot|None
        """
        return self.robots.get(name, None)

    def insert_robot(self, tree, pose, parents=None):
        """
        Inserts a robot into the world. This consists of two steps:

        - Sending the insert request message
        - Receiving a ModelInfo response

        This method is a coroutine because of the first step, writing
        the message must be yielded since PyGazebo doesn't appear to
        support writing multiple messages simultaneously. For the response,
        i.e. the message that confirms the robot has been inserted, a
        future is returned.

        :param parents:
        :param tree:
        :type tree: Tree
        :param pose:
        :type pose: Pose
        :return: A future that resolves with the created `Robot` object.
        """
        robot_id = self.get_robot_id()
        robot_name = "gen__" + str(robot_id)

        robot = tree.to_robot(robot_id)
        sdf = self.get_simulation_sdf(robot, robot_name)
        sdf.elements[0].set_pose(pose)

        if self.output_directory:
            with open(os.path.join(self.output_directory, 'robot_%d.sdf' % robot_id), 'w') as f:
                f.write(str(sdf))

        return_future = asyncio.Future()
        insert_future = self.insert_model(sdf)
        insert_future.add_done_callback(lambda fut: self._robot_inserted(
            robot_name, tree, robot, parents, fut.result(), return_future))

        asyncio.ensure_future(insert_future)
        return return_future

    def get_simulation_sdf(self, robot, robot_name):
        """

        :param robot:
        :type robot: PbRobot
        :param robot_name:
        :return:
        :rtype: SDF
        """
        raise NotImplementedError("Implement in subclass if you want to use this method.")

    def delete_robot(self, robot):
        """
        :param robot:
        :type robot: Robot
        :return:
        """
        # Immediately unregister the robot so no it won't be used
        # for anything else while it is being deleted.
        self.unregister_robot(robot)
        return self.delete_model(robot.name, req="delete_robot")

    def _robot_inserted(self, robot_name, tree, robot, parents, msg, return_future):
        """
        Registers a newly inserted robot and marks the insertion
        message response as handled.

        :param tree:
        :param robot_name:
        :param tree:
        :param robot:
        :param parents:
        :param msg:
        :type msg: pygazebo.msgs.response_pb2.Response
        :param return_future: Future to resolve with the created robot object.
        :type return_future: Future
        :return:
        """
        inserted = ModelInserted()
        inserted.ParseFromString(msg.serialized_data)
        model = inserted.model
        time = Time(msg=inserted.time)
        p = model.pose.position
        position = Vector3(p.x, p.y, p.z)

        robot = Robot(robot_name, tree, robot, position, time, parents)
        self.register_robot(robot)
        return_future.set_result(robot)

    def register_robot(self, robot):
        """
        Registers a robot with its Gazebo ID in the local array.
        :param robot:
        :type robot: Robot
        :return:
        """
        logger.debug("Registering robot %s." % robot.name)

        if robot.name in self.robots:
            raise ValueError("Duplicate robot: %s" % robot.name)

        self.robots[robot.name] = robot
        # if self.output_directory:
        #     # Write robot details and CSV row to files
        #     robot.write_robot('%s/robot_%d.pb' % (self.output_directory, robot.robot.id),
        #                       self.write_robots)

    def unregister_robot(self, robot):
        """
        Unregisters the robot with the given ID, usually happens when
        it is deleted.
        :param robot:
        :type robot: Robot
        :return:
        """
        logger.debug("Unregistering robot %s." % robot.name)
        del self.robots[robot.name]

    def _update_poses(self, msg):
        """
        Handles the pose info message by updating robot positions.
        :param msg:
        :return:
        """
        poses = poses_stamped_pb2.PosesStamped()
        poses.ParseFromString(msg)

        self.last_time = t = Time(msg=poses.time)
        if self.start_time is None:
            self.start_time = t

        for pose in poses.pose:
            robot = self.robots.get(pose.name, None)
            if not robot:
                continue

            position = Vector3(pose.position.x, pose.position.y, pose.position.z)
            # robot.update_position(t, position, self.write_poses)

        self.call_update_triggers()

    def add_update_trigger(self, callback):
        """
        Adds an update trigger, a function called every time the local
        state is updated.
        :param callback:
        :type callback: callable
        :return:
        """
        self.update_triggers.append(callback)

    def remove_update_trigger(self, callback):
        """
        Removes a previously installed update trigger.
        :param callback:
        :type callback: callable
        :return:
        """
        self.update_triggers.remove(callback)

    def call_update_triggers(self):
        """
        Calls all update triggers.
        :return:
        """
        for callback in self.update_triggers:
            callback(self)


from django.shortcuts import render
from django.utils.timesince import timesince
from .models import Scraper, Scraped
from django.contrib.auth.decorators import login_required
from django.http import HttpResponseRedirect
from django.contrib import messages
from django.utils import timezone

def front_page(request):
    return render(request, 'scraper/plain_post.html', {})

@login_required()
def simple_listing(request):
    scraper_data = []
    for scraper in Scraper.objects.filter(user=request.user):
        scraped = Scraped.objects.all().filter(scraper=scraper).latest("scraped_date")
        scraper_datum = {"name":scraper.name,
                         "scraper":scraper,
                         "user":scraper.user,
                         "timesince":timesince(scraped.scraped_date),
                         "scraped_date":scraped.scraped_date,
                         "last_data":scraped.data
                         }
        scraper_data.append(scraper_datum)
        scraper_data.sort(key=lambda scraped: scraped["scraped_date"], reverse=True)
    return render(request, 'scraper/test_post.html', {"scraper_data": scraper_data})

from .forms import NewScraper_1, fixed_freq_choices

@login_required()
def add_scraper_1(request):
    if request.method == 'POST':
        form = NewScraper_1(request.POST)
        if form.is_valid():

            new_scraper = Scraper(user=request.user,
                                  name=form.cleaned_data["name"],
                                  url=form.cleaned_data["url"],
                                  lxml=form.cleaned_data["lxml"],
                                  xhtml=form.cleaned_data["xhtml"],
                                  freq=fixed_freq_choices[form.cleaned_data["fixed_freq"]][0],
                                  freq_units=fixed_freq_choices[form.cleaned_data["fixed_freq"]][1],
                                  update_on_change=form.cleaned_data["update_on_change"] ,
                                  enabled=True)

            new_scraper.save()

            messages.success(request, "Scraper '%s' Created"%new_scraper.name)

            return HttpResponseRedirect('/manage_scrapers')

    # if a GET (or any other method) we'll create a blank form
    else:
        form = NewScraper_1()

    return render(request, 'scraper/new_scraper.html', {'form': form})

@login_required()
def manage_scrapers(request):
    scraper_data = []

    ## Whether to mark all as read
    markRead = True if request.GET.get("markRead")=="True" else False

    ## Ordering
    if request.GET.get("sortby") in ["name","url","last_updated","enabled"]:
        order_by = request.GET.get("sortby")
    else:
        order_by = "last_updated"

    if request.GET.get("order") == "ASC":
        order_by_descending = False
    else:
        order_by_descending = True

    final_order_by = "-" + order_by if order_by_descending else order_by

    timenow = timezone.now()

    for scraper in Scraper.objects.filter(user=request.user).order_by(final_order_by):
        scraper_datum = {"name":scraper.name,
                         "scraper":scraper,
                         "timesince":timesince(scraper.last_updated),
                         "freq_str":scraper.interval_to_str(),
                         }

        ## Implement marked all as read
        if markRead:
            scraper.last_viewed_by_user = timenow
            scraper.save()

        ## For now, just show if scraper has unviewed scrapeds
        if scraper.last_updated > scraper.last_viewed_by_user:
            scraper_datum["has_new"] = True
        else:
            scraper_datum["has_new"] = False

        scraper_data.append(scraper_datum)

    return render(request, 'scraper/manage_scrapers.html', {"scraper_data": scraper_data,
                                                            "order_by":order_by,
                                                            "order_by_descending":order_by_descending})

@login_required()
def view_scraper(request, scraper_id):
    scraper_ls = Scraper.objects.filter(id=scraper_id)
    if not scraper_ls:
        messages.info(request, 'Error viewing scraper')
        return manage_scrapers(request)

    scraper = scraper_ls[0]
    if scraper.user != request.user:
        messages.info(request, 'Error viewing scraper')
        return manage_scrapers(request)

    ## User just viewed scraper
    scraper.last_viewed_by_user = timezone.now()
    scraper.save()

    scraped_data = Scraped.objects.filter(scraper__id=scraper_id).order_by("-scraped_date")
    for scraped in scraped_data:
        scraped.timesince = timesince(scraped.scraped_date)

    scraper.freq_str = scraper.interval_to_str()
    scraper.timesince = timesince(scraper.last_updated)

    return render(request, 'scraper/view_scraper.html', {"scraped_data": scraped_data,
                                                         "scraper":scraper})

@login_required()
def delete_scraper(request, scraper_id):
    scraper_ls = Scraper.objects.filter(id=scraper_id)
    if not scraper_ls:
        messages.info(request, 'Error deleting scraper')
        return manage_scrapers(request)

    scraper = scraper_ls[0]
    if scraper.user != request.user:
        messages.info(request, 'Error deleting scraper')
        return manage_scrapers(request)

    if request.method == 'POST':
        scraper.delete()
        messages.success(request, "Scraper '%s' deleted"%scraper.name)
        return HttpResponseRedirect('/manage_scrapers')

    else:
        scraper.freq_str = scraper.interval_to_str()
        return render(request, 'scraper/delete_scraper.html', {"scraper": scraper})

@login_required()
def edit_scraper(request, scraper_id):
    scraper_ls = Scraper.objects.filter(id=scraper_id)
    if not scraper_ls:
        messages.info(request, 'Error viewing scraper')
        return manage_scrapers(request)

    scraper = scraper_ls[0]
    if scraper.user != request.user:
        messages.info(request, 'Error viewing scraper')
        return manage_scrapers(request)

    if request.method == 'POST':
        form = NewScraper_1(request.POST)
        if form.is_valid():

            scraper.user=request.user
            scraper.name=form.cleaned_data["name"]
            scraper.url=form.cleaned_data["url"]
            scraper.lxml=form.cleaned_data["lxml"]
            scraper.xhtml=form.cleaned_data["xhtml"]
            scraper.freq=fixed_freq_choices[form.cleaned_data["fixed_freq"]][0]
            scraper.freq_units=fixed_freq_choices[form.cleaned_data["fixed_freq"]][1]
            scraper.update_on_change=form.cleaned_data["update_on_change"]
            scraper.enabled=form.cleaned_data["enabled"]

            scraper.save()

            messages.success(request, "Scraper '%s' Modified"%scraper.name)

            return HttpResponseRedirect('/manage_scrapers')

    # if a GET (or any other method) we'll create a blank form
    else:

        fixed_freq_reverse_dict = {v: k for k, v in fixed_freq_choices.items()}
        fixed_freq = fixed_freq_reverse_dict.get((scraper.freq,scraper.freq_units),fixed_freq_choices.keys()[0])

        form = NewScraper_1({"name":scraper.name,
                             "url":scraper.url,
                             "lxml":scraper.lxml,
                             "xhtml":scraper.xhtml,
                             "fixed_freq":fixed_freq,
                             "update_on_change":scraper.update_on_change,
                             "enabled":scraper.enabled})

    return render(request, 'scraper/edit_scraper.html', {'form': form})


import numpy as np
from sklearn import preprocessing
#We imported a couple of packages. Let's create some sample data and add the line to this file:
input_data = np.array([[3, -1.5, 3, -6.4], [0, 3, -1.3, 4.1], [1, 2.3, -2.9, 4.3]])

data_standardized = preprocessing.scale(input_data)
print ("\nMean =", data_standardized.mean(axis=0))
print ("Std deviation =", data_standardized.std(axis=0))

data_scaler = preprocessing.MinMaxScaler(feature_range=(0, 1))
data_scaled = data_scaler.fit_transform(input_data)
print ("\nMin max scaled data =", data_scaled)

data_normalized = preprocessing.normalize(input_data, norm='l1')
print ("\nL1 normalized data =", data_normalized)

data_binarized = preprocessing.Binarizer(threshold=1.4).transform(input_data)
print ("\nBinarized data =", data_binarized)


#!/usr/bin/python
#
# runDocker.py - Functions to run and interact with the docker container.
#
# Copyright 2017 Clinc2Clound Team
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
# WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import docker
import tarfile
import tempfile
import os

client = docker.from_env()

CONTAINER_NAME = "ilent2/clinic2cloud"

INPUT_TARGET = "/home/neuro/"
OUTPUT_FILENAME = "output.mnc"
OUTPUT_TARGET = "/home/neuro/" + OUTPUT_FILENAME

def startDocker(dataSet):
    """ Start a new docker instance and copy the data into the container.

    @param dataSet      The folder name that container the input DICOM.
    @return container   Reference to the created docker container.
    """

    # Check for updates to the docker image
    client.images.pull(CONTAINER_NAME)

    container = client.api.create_container(CONTAINER_NAME)

    with docker.utils.tar(dataSet) as tar:
        client.api.put_archive(container, INPUT_TARGET, tar)

    client.api.start(container)

    return container

def waitUntilDone(container):
    """ Blocks until the docker container has processed the files.

    @param container    The container object returned by startDocker.
    @return jobStatus   Returns 0 if docker ran successfully.
    """

    return client.api.wait(container)

def checkIfDone(container):
    """ Checks if the docker has processed the files (non-blocking).
    To get the jobStatus you will need to call get status.

    @param container    The container object returned by startDocker.
    @return done        Returns True if stopped, False if running.
    """

    inspect = client.api.inspect_container(container['Id'])
    return inspect['State']['Running'] == False

def getStatus(container):
    """ Get the status of the docker job.

    @param container    The container object returned by startDocker.
    @return jobStatus   Returns 0 if docker ran successfully.
    """

    if not checkIfDone(container):
        raise Exception("Container not stopped.")

    inspect = client.api.inspect_container(container['Id'])
    return inspect['State']['ExitCode']

def finalizeJob(container, outputDir):
    """ Copy data out of docker and free resources.

    @param container    The container object returned by startDocker.
    @param outputDir    Directory name for the output MINC file.
    @return None
    """

    with tempfile.NamedTemporaryFile() as tmpfile:
        strm, stat = client.api.get_archive(container, OUTPUT_TARGET)

        for d in strm:
            tmpfile.write(d)
        tmpfile.seek(0)

        tar = tarfile.open(fileobj=tmpfile)
        tar.extractall(path=outputDir)
        tar.close()


from datetime import datetime
import dateutil
from comm.CommBase import CommBase
from comm.CommUtil import yuan_2_fen, fen_2_yuan
from error import ParamError
from init import Global
from utility import DataUtils, Logger

__author__ = 'Cedric Zhuang'

log = Logger.get_logger(__name__)

class Ticket(CommBase):

    MAXIMUM_VOLUME = 999900

    class Status(object):
        SUCCESS = 'success'
        FAILED = 'failed'
        INVALID = 'invalid_request'
        NONE = 'none'

    class Op(object):
        BUY = 'buy'
        SELL = 'sell'
        WITHDRAW = 'withdraw'
        NONE = 'none'

    def __init__(self):
        super(Ticket, self).__init__()
        self.type = self.TYPE_TICKET
        self.index = None
        self.place_at = None
        self.valid_till = None
        self.price = -1.0
        self.op = self.Op.NONE
        self.volume = 0
        # following properties are used by response
        self.status = self.Status.NONE
        self.deal_at = None

    def get_json(self):
        super(Ticket, self).get_json()
        self.add("status", self.status, None, self.Status.NONE)
        self.add("index", self.index, None)
        self.add("place_at", self.place_at, None)
        self.add("valid_till", self.valid_till, None)
        self.add("deal_at", self.deal_at, None)
        self.add("price", yuan_2_fen(self.price), -100, -1.0, None)
        self.add("op", self.op, None, self.Op.NONE)
        self.add("volume", self.volume, None, 0)

        return self.json()

    @classmethod
    def create(cls, stock_index=None, date=-1, price=-1.0, op=Op.BUY, volume=100):
        """
        create a ticket valid only in one day
        :param stock_index: index of the stock to buy or sell
        :param date: the day when the ticket is valid, in int format
        :param price: the price to deal, -1.0 means deal ASAP
        :param op: type of the operation, 'buy' or 'sell'
        :param volume: quantity of the stock, default to 100
        :return:
        """
        t = Ticket()

        if date is not None and date != -1:
            day = date % 100
            month = (date % 10000 - day) / 100
            year = int(date / 10000)
            t.place_at = datetime(year, month, day, 9, 0, 0)
            t.valid_till = datetime(year, month, day, 15, 30, 0)
        if price is None:
            price = -1.0
        t.price = price
        t.index = stock_index
        t.op = op

        if t.op != cls.Op.WITHDRAW:
            t.set_volume(volume)
        return t

    def set_volume(self, volume):
        if volume > self.MAXIMUM_VOLUME:
            log.warn('volume(%d) larger than maximum allowed, reset.', volume)
            volume = self.MAXIMUM_VOLUME
        if volume % 100 != 0:
            log.warn('volume(%d) should be the integer multiple of 100.', volume)
            volume -= volume % 100
        self.volume = volume

    @staticmethod
    def create_result(ticket_id, stock_index, status=Status.FAILED, deal_at=None, price=None, op='buy', volume=100):
        """
        create a ticket operation response
        :param ticket_id: id of the request ticket, must supply this.
        :param stock_index: stock index
        :param status: trade status of the ticket
        :param deal_at: deal timestamp(datetime).  NA if deal failed
        :param price: deal price.  NA if deal failed.
        :param op: buy or sell
        :param volume: volume of the stock.  NA if deal failed
        :return:
        """
        if ticket_id < 0:
            raise ParamError('ticket_id must be supplied')

        t = Ticket()
        t.type = CommBase.TYPE_TICKET_RESULT
        t.request_id = ticket_id
        t.index = stock_index
        t.status = status
        t.op = op
        if status == Ticket.Status.SUCCESS:
            t.deal_at = deal_at
        t.price = price
        t.volume = volume

        return t

    def __repr__(self):
        return self.get_json()

    @classmethod
    def parse_result(cls, response, store=None):
        """
        Process the ticket response from trader
        :param response:
        :param store:
        :return:
        """
        if store is None:
            store = Global
        stock = DataUtils.get_property(response, 'index', None)
        price = fen_2_yuan(DataUtils.get_property(response, 'price', -1))
        op = DataUtils.get_property(response, 'op', Ticket.Op.NONE)
        ticket_id = DataUtils.get_property(response, 'id', -1)
        volume = DataUtils.get_property(response, 'volume', -1)
        status = DataUtils.get_property(response, 'status', Ticket.Status.INVALID)
        deal_at = DataUtils.get_property(response, 'deal_at', None)
        if deal_at is not None:
            deal_at = dateutil.parser.parse(deal_at)

        if status not in [cls.Status.INVALID, cls.Status.NONE]:
            if status == cls.Status.SUCCESS:
                try:
                    if op == cls.Op.BUY:
                        store.add_stock(stock, volume, price, deal_at)
                    elif op == cls.Op.SELL:
                        store.sell_stock(stock, volume, price)
                except ParamError:
                    log.info("cannot %s %s at %d", op, stock, price)
            log.error("failed to %s %s at %d, status: %s", op, stock, price, status)
        else:
            log.error("request is not valid, status: %s", status)


from django.shortcuts import render
from django.utils import timezone
#models.py파일에 정의된 Post모델(Post class)을 가져올 거에요.
from .models import Post,Comment
from django.shortcuts import render, get_object_or_404

from .forms import PostForm,CommentForm
from django.shortcuts import redirect
from django.contrib.auth.decorators import login_required

# Create your views here.

#1. post_list view
#이 함수는 호출하여 받은(return) blog/post_list.html템플릿을 보여줍니다.
def post_list(request):
    #posts QuerySet을 템플릿 컨텍스트에 전달하는 것입니다.
    #posts라는 변수를 만들고 있다는 것을 기억하세요. 이 변수는 퀴리셋의 이름입니다.

    posts = Post.objects.filter(published_date__lte=timezone.now()).order_by('published_date')

    return render(request, 'blog/post_list.html', {'posts': posts})
#post_list를 뷰에서 보여주고 이를 템플릿에 전달하기 위해서는, 모델을 가져와야 합니다.
#Post모델에서 블로그 글을 가져오기 위해서는 쿼리셋(QuerySet)이 필요합니다.

# 목록이 들어있는 posts 변수를 템플릿tag에 넘겨주었습니다.

#2. post_detail view
def post_detail(request, pk):
    post = get_object_or_404(Post, pk=pk)

    return render(request, 'blog/post_detail.html', {'post': post})

#blog.views.post_detail는 post_detail 뷰 경로입니다.
#blog는 응용프로그램(디렉터리 blog)의 이름인 것을 꼭 기억하세요.
#views는 views.py파일명이에요.
#마지막 부분 post_detail는 view 이름입니다.

 #뷰에 매개변수 pk를 추가해봅시다. 뷰가 pk를 식별해야겠죠?
 #그래서 함수를 def post_detail(request, pk):
 #라고 정의합니다. urls(pk)과 동일하게 이름을 사용해야 합니다. 변수가 생략되면 오류가 날 거예요!

#블로그 게시글 한 개만 보려면, 아래와 같이 쿼리셋(queryset)을 작성해야해요.

#blog/views.py
#Post.objects.get(pk=pk)
#하지만 이 코드에는 문제가 있어요. 만약 해당 primary key(pk)의 Post를 찾지 못하면 오류가 나올 거에요!
#우리가 원하는게 아니죠!
#장고에는 이를 해결하기 위해 get_object_or_404라는 특별한 기능을 제공해요.
#pk에 해당하는 Post가 없을 경우, 멋진 페이지(페이지 찾을 수 없음 404 : Page Not Found 404)를 보여줄 거에요

#이제 어떻게 해야하는지 알고 있죠? 드디어 템플릿을 추가할 차례에요!

#Post 상세 페이지 템플릿 만들기
#blog/templates/blog 디렉터리 안에 post_detail.html라는 새 파일을 생성하고 아래와 같이 코드를 작성하세요

#3.post_new view
#in base.html , 부트스트랩 테마에 있는 glyphicon glyphicon-plus 클래스로 더하기 기호가 보이게 되는데요.
@login_required
def post_new(request):
    if request.method == "POST":
        form = PostForm(request.POST)
        if form.is_valid():
            post = form.save(commit=False)
            post.author = request.user
            post.save()
            #로그 글을 작성한 다음에 post_detail페이지로 이동할 수 있으면 좋겠죠?
            #ost_detail 뷰 는 pk변수가 필요한 거 기억하고 있겠죠?
            # pk=post.pk를 사용해서 뷰에게 값을 넘겨줄 거에요.
            #여기서 post는 새로 생성한 블로그 글이에요.
            return redirect('post_detail', pk=post.pk)
    else:
        form = PostForm()
    return render(request, 'blog/post_edit.html', {'form': form})
#새 Post 폼을 추가하기 위해 PostForm() 함수를 호출하도록 하여 템플릿에 넘깁니다. 곧 view 로 다시 돌아와서 이 작업을 하겠지만, 지금 당장은 폼을 위한 템플릿을 먼저 빨리 만들어보도록 할게요
#template == html
@login_required
def post_publish(request, pk):
    post = get_object_or_404(Post, pk=pk)
    post.publish()
    return redirect('post_detail', pk=pk)
@login_required
def post_remove(request, pk):
    post = get_object_or_404(Post, pk=pk)
    post.delete()
    return redirect('post_list')

#을 제출할 때, 같은 뷰를 불러옵니다. 이때 request에는 우리가 입력했던 데이터들을 가지고 있는데, request.POST가 이 데이터를 가지고 있습니다. (POST는 글 데이터를 "등록하는(posting)"하는 것을 의미합니다. 블로그 "글"을 의미하는 "post"와 관련이 없어요)
# HTML에서 <form>정의에 method="POST"라는 속성이 있던 것이 기억나나요?
#이렇게 POST로 넘겨진 폼 필드의 값들은 이제 request.POST에 저장됩니다.
#이제 view 에서 두 상황으로 나누어 처리해볼게요.

#첫 번째: 처음 페이지에 접속했을 때입니다. 당연히 우리가 새 글을 쓸 수 있게 폼이 비어있어야겠죠.
#두 번째: 폼에 입력된 데이터를 view 페이지로 가지고 올 때입니다. 여기서 조건문을 추가시켜야 해요. (if를 사용하세요)

#if request.method == "POST":
    #[...]
#else:
    #form = PostForm()

#이제 생략된 [...]부분에 코드를 추가해봅시다. 만약 method가 POST라면, 폼에서 받은 데이터를 PostForm으로 넘겨줘야겠죠?

def add_comment_to_post(request, pk):
    post = get_object_or_404(Post, pk=pk)
    if request.method == "POST":
        form = CommentForm(request.POST)
        if form.is_valid():
            comment = form.save(commit=False)
            comment.post = post
            comment.save()
            return redirect('post_detail', pk=post.pk)
    else:
        form = CommentForm()
    return render(request, 'blog/add_comment_to_post.html', {'form': form})

@login_required
def comment_approve(request, pk):
    comment = get_object_or_404(Comment, pk=pk)
    comment.approve()
    return redirect('post_detail', pk=comment.post.pk)

@login_required
def comment_remove(request, pk):
    comment = get_object_or_404(Comment, pk=pk)
    comment.delete()
    return redirect('post_detail', pk=comment.post.pk)

@login_required
def post_edit(request, pk):
    post = get_object_or_404(Post, pk=pk)
    if request.method == "POST":
        form = PostForm(request.POST,instance=post)
        if form.is_valid():
            post = form.save(commit=False)
            post.author = request.user
            post.save()
            return redirect('post_detail', pk=post.pk)
    else:
        form = PostForm(instance=post)
    return render(request, 'blog/post_edit.html', {'form': form})

#첫 번째: url로부터 추가로 pk 매개변수를 받아서 처리합니다.
#두 번째: get_object_or_404(Post, pk=pk)를 호출하여 수정하고자 하는 글의
#Post 모델 인스턴스(instance)로 가져옵니다.
#(pk로 원하는 글을 찾습니다) 이렇게 가져온 데이터를
#폼을 만들 때와(글을 수정할 때 폼에 이전에 입력했던 데이터가 있어야
# 하겠죠?) 폼을 저장할 때 사용하게 됩니다
@login_required
def post_draft_list(request):
    posts = Post.objects.filter(published_date__isnull=True).order_by('created_date')
    return render(request, 'blog/post_draft_list.html', {'posts': posts})

def post_about(request):
    #posts QuerySet을 템플릿 컨텍스트에 전달하는 것입니다.
    #posts라는 변수를 만들고 있다는 것을 기억하세요. 이 변수는 퀴리셋의 이름입니다.
    return render(request, 'blog/post_about.html')

def post_it(request):
    posts = Post.objects.filter(category=1)
    return render(request,"blog/post_it.html",{'posts': posts})

def post_daily(request):
    posts = Post.objects.filter(category=2)
    return render(request,"blog/post_daily.html",{'posts': posts})

def post_theqoo(request):
    posts = Post.objects.filter(category=3)
    return render(request,"blog/post_theqoo.html",{'posts': posts})

def post_guu(request):
    posts = Post.objects.filter(category=4)
    return render(request,"blog/post_guu.html",{'posts': posts})


import requests
import bs4
import csv
import os
import smtplib
import configs
from email.message import EmailMessage

class Movie(object):
    movieList = []

    def __init__(self, imdbID, url, title, genre, plot, score, image):
        self.imdbID = imdbID
        self.url = url
        self.title = title
        self.genre = genre
        self.plot = plot
        self.score = score
        self.image = image
        Movie.movieList.append(self)

def makeSoup(source):
    res = requests.get(source)
    res.raise_for_status()
    return bs4.BeautifulSoup(res.text, 'lxml')

def buildTable():
    # opens csv to append new movies into after table entry is build
    with open('pastReleases.csv', 'a') as writer:
        newReleaseWriter = csv.writer(writer)
        # opens csv to read existing entries
        with open('pastReleases.csv', 'r') as reader:
            pastReleaseReader = csv.reader(reader)
            pastReleases = []
            for row in pastReleaseReader:
                pastReleases.append(row)
            # builds table by checking if movie has already been scraped
            table = ''
            for movie in Movie.movieList:
                if ([movie.title]) not in pastReleases:
                    tableEntry = f"""\

        {movie.title}

        {movie.genre}

        IMDB Score: {movie.score}

        {movie.plot}

                    """
                    table += tableEntry
                    newReleaseWriter.writerow([movie.title])
            return table

def scrape():
    # scrapes info and creates Movie object for each movie
    for movie in imdbSoup.find_all('div', class_="lister-item mode-detail"):
        imdbID = movie.div['data-tconst']  # access tags like dicts
        url = f'https://www.imdb.com/title/{imdbID}'
        title = movie.h3.a.text.strip()
        genre = movie.find('span', class_="genre").text.strip()
        plot = movie.find('p', class_="").text.strip()
        score = movie.find(
            'span', class_="ipl-rating-star__rating").text.strip()
        image = movie.find('img', class_="loadlate")['loadlate']
        # create a movie object for each movie, accessible in Movie.movieList
        Movie(imdbID, url, title, genre, plot, score, image)
        # print(title)

def sendEmail(movies):
    msg = EmailMessage()
    msg['Subject'] = f"Recent Movie Releases"
    msg['From'] = configs.sender
    msg['To'] = configs.recipients
    msg.add_alternative(f"""\
<!DOCTYPE html>

<meta content="en-us" http-equiv="Content-Language">

Recent Releases:

        {moviesTable}

""", subtype='html')

    if moviesTable:  # if movie table is empty, email does not send
        with smtplib.SMTP_SSL('smtp.gmail.com', 465) as smtp:
            smtp.login(configs.sender, configs.password)
            smtp.send_message(msg)

# main
# makes beautiful soup using IMDB New Releases page
imdbSoup = makeSoup(
    'https://www.imdb.com/list/ls016522954/?ref_=ttls_ref_typ&sort=release_date,desc&st_dt=&mode=detail&page=1&title_type=movie')

scrape()
moviesTable = buildTable()
sendEmail(moviesTable)


from typing import Dict, Any
from argparse import Namespace
import os
import shutil
import random
import numpy as np
import torch
import torch.nn as nn
import json

SAVE_DIR = 'checkpoints'
CONFIG_DIR = 'configs'

class Arguments:
    def __init__(self, args):
        self.__dict__ = args

    def __repr__(self):
        return str(self.__dict__)

def json_to_args(filename):
    with open(os.path.join(CONFIG_DIR, filename + '.json')) as f:
        return Arguments(json.load(f))

def get_run_name(args: Namespace, save_dir: str = SAVE_DIR) -> str:
    if not os.path.isdir(save_dir):
        os.makedirs(save_dir)

    if args.checkpoint:
        return os.path.join(save_dir, args.checkpoint)

    if args.name:
        return os.path.join(save_dir, args.name)

    dirlist = [f for f in os.listdir(save_dir) if os.path.isdir(os.path.join(save_dir, f))]
    dirlist.sort()
    dirlist.sort(key=lambda k: (len(k), k))  # Sort alphabetically but by length
    if not dirlist:
        result = 'A'
    else:
        last_run_char = dirlist[-1][-1]
        if last_run_char == 'Z':
            result = 'A' * (len(dirlist[-1])+1)
        else:
            result = dirlist[-1][:-1] + chr(ord(last_run_char) + 1)
    out_dir = os.path.join(save_dir, result)
    os.makedirs(out_dir)
    return out_dir

def set_rng_state(checkpoint):
    if checkpoint:
        random.setstate(checkpoint['rng_state'])
        np.random.set_state(checkpoint['np_rng_state'])
        torch.set_rng_state(checkpoint['torch_rng_state'])

def save_checkpoint(state: Dict[str, Any], run_name: str, is_best: bool) -> None:
    """ Saves model and training parameters at checkpoint + 'last.pth.tar'.
    If is_best is True, also saves best.pth.tar
    Args:
        state: (dict) contains model's state_dict, may contain other keys such as
        epoch, optimizer_state_dict
        run_name: (string) folder where parameters are to be saved
        is_best: (bool) True if it is the best model seen till now
    """
    print('Saving checkpoint...')
    save_path = os.path.join(run_name, 'checkpoint.pth.tar')
    torch.save(state, save_path)
    if is_best:
        print('Saving new model_best...')
        shutil.copyfile(save_path, os.path.join(run_name, 'model_best.pth.tar'))

def load_checkpoint(checkpoint_name: str, use_best: bool = False) -> Dict[str, Any]:
    """ Loads torch checkpoint.
    Args:
        checkpoint: (string) filename which needs to be loaded
    """
    if not checkpoint_name:
        return {}
    print('Loading checkpoint...')
    load_file = 'model_best.pth.tar' if use_best else 'checkpoint.pth.tar'
    return torch.load(os.path.join(SAVE_DIR, checkpoint_name, load_file))

def load_state_dict(checkpoint: Dict, model: nn.Module, optimizer=None):
    """ Loads model parameters (state_dict) from checkpoint. If optimizer is provided,
    loads state_dict of optimizer assuming it is present in checkpoint.
    Args:
        checkpoint: () checkpoint object
        model: (torch.nn.Module) model for which the parameters are loaded
        optimizer: (torch.optim) optional: resume optimizer from checkpoint
    """
    if checkpoint:
        model.load_state_dict(checkpoint['state_dict'])
        if optimizer is not None:
            optimizer.load_state_dict(checkpoint['optimizer_state_dict'])


import pygame
import numpy as numpy

pygame.init()

width, height = 500, 500
screen = pygame.display.set_mode((height, width))

bg = 25, 25, 25
screen.fill(bg)

nxC, nyC = 10, 10

dimCW = width / nxC
dimCH = height / nyC

while True:

    for y in range(0, nxC):
        for x in range(0, nyC):

            poly = [((x) * dimCW,  y * dimCH),
                    ((x+1) * dimCW,  y  * dimCH),
                    ((x+1) * dimCW, (y+1) * dimCH),
                    ((x)  * dimCW, (y+1) * dimCH)]

            pygame.draw.polygon(screen, (128, 128, 128), poly, 1)

    pygame.draw.circle(screen, (255, 0, 0), (250, 250), 5)

    pygame.display.flip()

    pass


import requests
from decouple import config

def send_message(text):
    token = config('TOKEN')
    api_url = f'https://api.telegram.org/bot{token}'

    # 내가 챗봇에 보낸 메세지를 통해 나의 id를 알아내고 내 id로 메세지를 보낸다.
    updates = requests.get(api_url + '/getUpdates').json()  # 일정시간이 지나면 업데이트 내역에 삭제된다.
    print(updates)
    chat_id = updates['result'][0]['message']['from']['id']

    requests.get(api_url + f'/sendMessage?chat_id={chat_id}&text={text}')

    print(chat_id)


# A significant amount of the code was taken from this website:
# https://programminghistorian.org/en/lessons/creating-apis-with-python-and-flask
# Some of the consumer() function below was taken from this site:
# https://www.programiz.com/python-programming/examples/fibonacci-sequence
# This will only be available on the 127.0.0.1 IP address from the server it is running on.
# You could change 127.0.0.1 to 0.0.0.0. Then it would be available to external web taffic, but this is not a secure way of doing it.

import flask
import json
from flask import request, jsonify

app = flask.Flask(__name__)
app.config["DEBUG"] = True

# Create some test data for our catalog in the form of a list of dictionaries.
defaultdat = [
    { 'val1': 100,
      'val2': 200,
      'val3': 300},
    { 'val1': 111,
      'val2': 222,
      'val3': 333},
    { 'val1': 1,
      'val2': 2,
      'val3': 3}
]

@app.route('/', methods=['GET'])
def home():
    return '''
A RESTful API Program Powered by Flask and Python

This is a basic API for displaying the Fibonacci sequence based on an integer value provided through a POST.
'''

# A route to return the default values.
@app.route('/api/v1/resources/defaultdat/all', methods=['GET'])
def api_all():
    return jsonify(defaultdat)

@app.route('/api/v1/resources/fibonacci', methods=['GET', 'POST'])
def consumer():
    data = "You did not enter an integer.  Please only enter integers greater than 1."
    n1 = 0
    n2 = 1
    count = 0
    builder = [0]
    if request.method == 'POST':
        stest = str(request.form['InputValue'][1:-1])
        if (stest[0] == '-'):
          stest = stest[1:len(stest)]
        justnum = stest.isnumeric()
        if(justnum):
          nterms = int(request.form['InputValue'][1:-1])
          if nterms <= 0:
             count = "Please enter a positive integer"
             print("Please enter a positive integer")
          elif nterms == 1:
             print("Fibonacci sequence up to",nterms,":")
             print(n1)
             count = str(count) + ", " + str(n1)
          else:
             print("Fibonacci sequence:")
             while count < nterms:
                 nth = n1 + n2
                 # update values
                 n1 = n2
                 n2 = nth
                 count += 1
                 builder.append(str(n1))

          builder = str(builder)
          if len(builder) > 1:
              data = '''
The Fibonacci sequence for the number you entered is ''' + builder[0:-1] + "]" + '''
                                                                       '''
          else:
            data = "  \
Please enter an integer higher than 1.                   \
                          "
        return data

app.run(host="127.0.0.1", port="5050")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Jan 23 10:32:18 2020

@author: j-bd
"""

import logging

import numpy as np

from perceptron import Perceptron

logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG)

'''Evaluating the Perceptron Bitwise Datasets with the XOR dataset'''

# construct the XOR dataset
XS = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])
YS = np.array([[0], [1], [1], [0]])

# define our perceptron and train it
logging.info(" Training perceptron...")
P = Perceptron(XS.shape[1], alpha=0.1)
P.fit(XS, YS, epochs=20)

# now that our perceptron is trained we can evaluate it
logging.info(" Testing perceptron...")

# now that our network is trained, loop over the data points
for (x, target) in zip(XS, YS):
    # make a prediction on the data point and display the result
    # to our console
    pred = P.predict(x)
    logging.info(f" Data={x}, ground-truth={target[0]}, pred={pred}")


from django.shortcuts import render
from django.http import JsonResponse
from django.views import generic
from django.forms.models import model_to_dict
from rest_framework.views import APIView
from rest_framework import status
from bs4 import BeautifulSoup
from lxml import html
from .models import *
from .serializers import *
import json
import re
import requests
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
from django.utils.encoding import smart_str
from urllib import parse

class ApiIndexView(APIView):

    def get(self,request,*args, **kwargs):
        return JsonResponse({'status':"OK"})

    def clean_data(self):

        data = {}

        for k, v in self.request.data.items():
            clean = re.compile('>.*?<')
            texts = re.sub(clean,'><',v)
            data.update({k:texts})
        return data

    def get_attribute_from_html(self, data):
        result = {}
        tag_name = []
        for key, value in data.items():
            if "url" != key:
                pars = BeautifulSoup(value, "html.parser").find()
                if pars.get("id", False):
                    parsed_data = "#" + pars.get("id")
                    result[key] = parsed_data
                elif pars.get("src", False):
                    parsed_data = "@" + pars.get("src")
                    result[key] = parsed_data + '|' + pars.name
                elif pars.get("class", False):
                    parsed_data = "." + " ".join(pars.get("class"))
                    result[key] = parsed_data
                elif pars.get('itemprop', False):
                    parsed_data = "&" + pars.get("itemprop")
                    result[key] = parsed_data + "|" + pars.name
                elif pars.get('href', False):
                    parsed_data = "^" + pars.get("href")
                    result[key] = parsed_data + "|" + pars.name
                else:
                    tag_name.append(key)
            else:
                result[key] = value
        if tag_name:
            return [result, tag_name]
        return result

    def post(self, request, *args, **kwargs):

        clean_data = self.clean_data()
        data = self.get_attribute_from_html(clean_data)

        if isinstance(data, dict):
            product = ProductTag.objects.create()
            product.name = data.get("url")
            product.bulk_insert(**data)
            try:
                product.save()
            except:
                print(data)
            return JsonResponse({'data': data})
        else:
            result = data[0]

            for tag in data[1]:
                result[tag] = self.request.data[tag]
            product = ProductTag.objects.create()
            print(result.get("url"))
            product.name = result.get("url")
            product.bulk_insert(**result)
            product.save()
            return JsonResponse({'data': result})

class ScraperView(generic.View):

    def scrapper(self, url, data):
        result = {}
        options = Options()
        options.headless = True
        driver = webdriver.Firefox(options=options)
        driver.get(url)
        pars = BeautifulSoup(driver.page_source, "html.parser")
        for obj in data:
            key, value, tag_name = obj.get("field"), obj.get("value"), obj.get("tag_name")

            if value.startswith("."):

                result[key] = pars.find_all(tag_name, {"class": value[1:]})[0].text if pars.find_all(tag_name, {"class": value[1:]}) else "yoxdu"

            elif value.startswith("@"):
                result[key] = driver.find_element_by_xpath('//*[@src="{}"]'.format(value[1:])).get_attribute("src")

            elif value.startswith("&"):
                result[key] = driver.find_element_by_xpath('//*[@itemprop="{}"]'.format(value[1:])).text

            elif value.startswith("^"):
                result[key] = driver.find_element_by_xpath('//*[@href="{}"]'.format(value[1:])).text

            elif value.startswith("<"):
                pars = BeautifulSoup(value, "html.parser").find()

                result[key] = driver.find_elements_by_xpath("//{}".format(pars.name))[0].text

            elif value.startswith("#"):
                result[key] = pars.find_all(tag_name, {"id": value[1:]})[0].text if pars.find_all(tag_name, {"id": value[1:]}) else "yoxdu"

        driver.quit()
        return result

    def get(self, request, *args, **kwargs):

        url = request.GET.get("url")
        prs_url = parse.urlsplit(url)

        data = ProductTag.objects.filter(name__icontains=prs_url.netloc).last()
        obj = ProductSerializers(data)
        latest_result = self.scrapper(url, obj.data.get("product_tags"))
        return JsonResponse({"status":"OK", "data": latest_result})


import glob
import os
import shutil
import argparse
import numpy as np
import cv2
from tqdm import tqdm
import platform

parser = argparse.ArgumentParser()
parser.add_argument('--imgs_dir', type=str, default='', help='imgs dir.')
parser.add_argument('--center_file', type=str, default='', help='file to define eval centers.')
parser.add_argument('--out_dir_base', type=str, default='./fake_data', help='output dir.')
args = parser.parse_args()

if platform.system().lower()=="linux":
    split_symbol = '/'
elif platform.system().lower()=="windows":
    split_symbol = '\\'
else:
    raise ValueError('Do not support!!!')

NC=3
IMG_SIZE=64
RADIUS=2 # the unit is degree

def get_file_list(dataset_dir):
    file_list = glob.glob(os.path.join(dataset_dir, '*.png')) + glob.glob(os.path.join(dataset_dir, '*.jpg'))
    file_list.sort()
    return file_list

### image directory
imgs_dir = args.imgs_dir
output_dir_base = os.path.join(args.out_dir_base, 'centers')
print(output_dir_base)
os.makedirs(output_dir_base, exist_ok=True)

### load all images and labels first
img_lists = get_file_list(imgs_dir)
print("{} fake images".format(len(img_lists)))
n_img = len(img_lists)
images = np.zeros((n_img, IMG_SIZE, IMG_SIZE, NC))
labels = np.zeros(n_img)

print("Start loading images and labels...")
for i in tqdm(range(n_img)):
    fullpath_i = img_lists[i]
    filename_i = fullpath_i.split(split_symbol)[-1]
    label_i = float((filename_i.split('.png')[0]).split('_')[-1])
    labels[i] = label_i
    images[i] = cv2.imread(fullpath_i)
#end for i

### load centers
centers = np.loadtxt(args.center_file)
num_centers = len(centers)

for i in range(num_centers):
    center_i = centers[i]
    lb_i = center_i - RADIUS
    ub_i = center_i + RADIUS
    indx_i = np.where((labels>=lb_i)*(labels<=ub_i)==True)[0]
    images_indx_i = images[indx_i]
    labels_indx_i = labels[indx_i]

    print('\r {}/{}, center={}, lb={}, ub={}, num_img={}.\n'.format(i, num_centers, center_i, lb_i, ub_i, len(indx_i)))

    output_dir = os.path.join(output_dir_base, str(i+1)) #the center ID is from 1 to num_centers
    os.makedirs(output_dir, exist_ok=True)

    for j in range(len(indx_i)):
        image_j_indx_i = images_indx_i[j]
        filename_i = os.path.join(output_dir, '{}_{}.png'.format(j, labels_indx_i[j]))
        cv2.imwrite(filename_i, image_j_indx_i)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
协程：利用一个线程，分解一个线程为多个"微线程"   -----程序级别实现的，跟系统无无关
协程在IO操作时使用才能提高效率

模块：greenlet  gevent ---------gevent是对greenlet的封装（greenlet实现协程功能）

使用模块：pip3 install gevent
"""

"""
#使用greenlet
from gevent.greenlet import greenlet

def f1():
    print(12)
    gr2.switch()
    print(34)
    gr2.switch()

def f2():
    print(56)
    gr1.switch()
    print(78)

gr1 = greenlet(f1)
gr2 = greenlet(f2)
gr1.switch()

"""

"""
#使用gevnet
import gevent

def foo():
    print('running in foo')
    gevent.sleep(0)
    print('Explicit context switch to foo again')

def bar():
    print('Exlicit context to bar')
    gevent.sleep(0)
    print('Implicit context switch back to bar')

gevent.joinall([
    gevent.spawn(foo),
    gevent.spawn(bar),
])
"""

# 协程应用示例：
# 协程发送先请求给f，而不等其执行完成，就发下一个
# 先把所有请求发送出去，然后再等待其执行完成（就好像一个线程又创建了多个微线程来执行任务）
from gevent import monkey;monkey.patch_all()   #默认socket是不支持执行完成通知的功能，这个就需要用monkey来封装一个具有通知功能的socket
                                                # ---没有这个，协程执行任务时还是等一个任务执行完才会创建并执行下一个任务
import gevent
import requests

def f(url):
    print('GET: ',url)
    response = requests.get(url)
    data = response.text
    print('%d received from %s' % (len(data),url))

# 批量创建任务，而不等待执行完成，最后统一等待执行结果
gevent.joinall([
    gevent.spawn(f, 'https://www.python.org/'),
    gevent.spawn(f, 'https://www.baidu.com/'),
    gevent.spawn(f, 'https://www.github.com/'),
])