import sys
import os
import os.path
from flask import Flask
from flask_s3_viewer import FlaskS3Viewer
from flask_s3_viewer.aws.ref import Region

import subprocess
from os import remove
import sys

if os.getenv("NB_PREFIX"):
    PREFIX = os.getenv("NB_PREFIX")
else:
    PREFIX = ""

app = Flask(__name__)

#app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER

if PREFIX:
    from flask import url_for, send_from_directory
    from flask_reverse_proxy_fix.middleware import ReverseProxyPrefixFix
    app.config['REVERSE_PROXY_PATH'] = PREFIX + '/proxy/8765'
    ReverseProxyPrefixFix(app)

def _get_minio_client(tenant):
    """ Get the variables out of vault to create Minio Client. """

    d = {
        'MINIO_URL'        : None,
        'MINIO_ACCESS_KEY' : None,
        'MINIO_SECRET_KEY' : None
    }

    if tenant not in ("minimal", "premium", "pachyderm"):
        print("Not a valid resource! Options are")
        print("minimal, premium, pachyderm.")
        print("We will try anyway...")

    vault = f"/vault/secrets/minio-{tenant}-tenant1"

    for var in d:
        CMD = f'echo $(source {vault}; echo ${var})'
        p = subprocess.Popen(CMD, stdout=subprocess.PIPE, shell=True,
                             executable='/bin/bash')
        d[var] = p.stdout.readlines()[0].strip().decode("utf-8")

    return d

def get_minimal_client():
    """Get a connection to the minimal Minio tenant"""
    return __get_minio_client__("minimal")

def get_pachyderm_client():
    """Get a connection to the pachyderm Minio tenant"""
    return __get_minio_client__("pachyderm")

def get_premium_client():
    """Get a connection to the premium Minio tenant"""
    return __get_minio_client__("premium")

if len(sys.argv) > 1:
    tenant = sys.argv[0]
else:
    tenant = "minimal"

d = _get_minio_client(tenant)

NB_PREFIX = os.getenv("NB_PREFIX") + '/proxy/8765'
BUCKET = os.getenv("NB_PREFIX").split('/')[2]

app.config['DOWNLOAD_DIR'] = os.path.join(os.getenv("HOME"), "downloads")

creds_dir = os.path.join(os.getenv("HOME"), ".aws")
os.makedirs(creds_dir, exist_ok=True)

# These don't have to be correct!! Anything works.
with open(creds_dir + "/credentials", "w") as f:
    f.write("""
[minio]
aws_access_key_id=minioadmin
aws_secret_access_key=minioadmin
region = us-east-1
""")

# # For test, disable template caching
# app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 1
# app.config['TEMPLATES_AUTO_RELOAD'] = True

# FlaskS3Viewer Init
s3viewer = FlaskS3Viewer(
    app, # Flask app
    namespace=BUCKET, # namespace be unique
    template_namespace='mdl', # set template
    object_hostname='http://flask-s3-viewer.com', # file's hostname
    allowed_extensions={}, # allowed extension
    config={ # Bucket configs and else
        'profile_name': 'minio',
        'access_key': d['MINIO_ACCESS_KEY'],
        'secret_key': d['MINIO_SECRET_KEY'],
        'region_name': Region.SEOUL.value,
        'endpoint_url': d['MINIO_URL'],
        'bucket_name': BUCKET,
        'use_ssl': False,
        'cache_dir': '/tmp/flask_s3_viewer',
        'use_cache': False,
        'ttl': 86400
    }
)

# Init another one
s3viewer.add_new_one(
    namespace='shared',
    template_namespace='mdl', # set template
    object_hostname='http://example.com',
    config={
        'profile_name': 'minio',
        'access_key': d['MINIO_ACCESS_KEY'],
        'secret_key': d['MINIO_SECRET_KEY'],
        'endpoint_url': d["MINIO_URL"],
        'bucket_name': 'shared',
        'use_ssl': False,
        'region_name': Region.SEOUL.value
    }
)

# You can see registerd configs
# print(s3viewer.FLASK_S3_VIEWER_BUCKET_CONFIGS)

# You can use boto3's session and client if you want
# print(FlaskS3Viewer.get_boto_client(FS3V_NAMESPACE))
# print(FlaskS3Viewer.get_boto_session(FS3V_NAMESPACE))

# Apply FlaskS3Viewer blueprint
s3viewer.register()

@app.route('/')
def index ():
    return f"""
This is the root of the storage explorer! We will add a nice splash page later.

You probably want to go to one of these:

{BUCKET}

Shared

"""

print(f"""

Thanks for using the DAaaS Jupyter Storage System! Go to

https://kubeflow.covid.cloud.statcan.ca{NB_PREFIX}

""")

# Usage: python example.py test (run debug mode)
if __name__ == '__main__':
    debug = False
    if len(sys.argv) > 1:
        if sys.argv[1] == 'test':
            debug = True
    app.run(debug=debug, port=8765)


# -----------------------------------------------------------------------------
# From Numpy to Python
# Copyright (2017) Nicolas P. Rougier - BSD license
# More information at https://github.com/rougier/numpy-book
# -----------------------------------------------------------------------------

import numpy as np
import numba as nb

@nb.jit(nopython=True, parallel=True, fastmath=True)
def linspace(start, stop, num, dtype):
    X = np.empty((num, ), dtype=dtype)
    dist = (stop - start) / (num - 1)
    for i in range(num):
        X[i] = start + i * dist
    return X

@nb.jit(nopython=True, parallel=True, fastmath=True)
def mandelbrot(xmin, xmax, ymin, ymax, xn, yn, maxiter, horizon=2.0):
    # Adapted from https://www.ibm.com/developerworks/community/blogs/jfp/...
    #              .../entry/How_To_Compute_Mandelbrodt_Set_Quickly?lang=en
    # X = np.linspace(xmin, xmax, xn, dtype=np.float64)
    # Y = np.linspace(ymin, ymax, yn, dtype=np.float64)
    X = linspace(xmin, xmax, xn, dtype=np.float64)
    Y = linspace(ymin, ymax, yn, dtype=np.float64)
    # C = X + Y[:,None]*1j
    C = X + np.reshape(Y, (yn, 1)) * 1j
    N = np.zeros(C.shape, dtype=np.int64)
    Z = np.zeros(C.shape, dtype=np.complex128)
    for n in range(maxiter):
        # I = np.less(abs(Z), horizon)
        I = np.less(np.absolute(Z), horizon)
        # N[I] = n
        for j in range(C.shape[0]):
            for k in range(C.shape[1]):
                if I[j, k]:
                    N[j, k] = n
        # Z[I] = Z[I]**2 + C[I]
        for j in range(C.shape[0]):
            for k in range(C.shape[1]):
                if I[j, k]:
                    Z[j, k] = Z[j, k]**2 + C[j, k]
    # N[N == maxiter-1] = 0
    for j in range(C.shape[0]):
        for k in range(C.shape[1]):
            if N[j, k] == maxiter - 1:
                N[j, k] = 0
    return Z, N


# -*- coding: utf-8 -*-

import os
import sys
from setuptools import setup, find_packages, Extension
from distutils.errors import (CCompilerError, DistutilsExecError,
                              DistutilsPlatformError)
from distutils.command.build_ext import build_ext

try:
    FileNotFoundError
except NameError:
    FileNotFoundError = IOError  # py2k

def get_version():
    basedir = os.path.dirname(__file__)
    with open(os.path.join(basedir, 'pendulum/version.py')) as f:
        variables = {}
        exec(f.read(), variables)

        version = variables.get('VERSION')
        if version:
            return version

    raise RuntimeError('No version info found.')

__version__ = get_version()

# C Extensions
with_extensions = os.getenv('PENDULUM_EXTENSIONS', None)

if with_extensions == '1' or with_extensions is None:
    with_extensions = True

if with_extensions == '0' or hasattr(sys, 'pypy_version_info'):
    with_extensions = False

extensions = []
if with_extensions:
    extensions = [
        Extension('pendulum._extensions._helpers',
                  ['pendulum/_extensions/_helpers.c'],
                  extra_compile_args=['-Wno-unused-function']),
    ]

class BuildFailed(Exception):

    pass

class ve_build_ext(build_ext):
    # This class allows C extension building to fail.

    def run(self):
        try:
            build_ext.run(self)
        except (DistutilsPlatformError, FileNotFoundError):
            raise BuildFailed()

    def build_extension(self, ext):
        try:
            build_ext.build_extension(self, ext)
        except (CCompilerError, DistutilsExecError,
                DistutilsPlatformError, ValueError):
            raise BuildFailed()

kwargs = dict(
    name='pendulum',
    license='MIT',
    version=__version__,
    description='Python datetimes made easy.',
    long_description=open('README.rst').read(),
    author='Sébastien Eustace',
    author_email='sebastien@eustace.io',
    url='https://github.com/sdispater/pendulum',
    download_url='https://github.com/sdispater/pendulum/archive/%s.tar.gz' % __version__,
    packages=find_packages(exclude=['tests']),
    install_requires=[
        'tzlocal',
        'python-dateutil',
        'pytzdata',
    ],
    include_package_data=True,
    tests_require=['pytest'],
    test_suite='nose.collector',
    classifiers=[
        'Intended Audience :: Developers',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.2',
        'Programming Language :: Python :: 3.3',
        'Programming Language :: Python :: 3.4',
        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: 3.6',
        'Programming Language :: Python :: Implementation :: CPython',
        'Programming Language :: Python :: Implementation :: PyPy'
    ],
)

if extensions:
    kwargs['ext_modules'] = extensions
    kwargs['cmdclass'] = dict(build_ext=ve_build_ext)

try:
    setup(**kwargs)
except BuildFailed:
    print("************************************************************")
    print("Cannot compile C accelerator module, use pure python version")
    print("************************************************************")
    del kwargs['ext_modules']
    del kwargs['cmdclass']
    setup(**kwargs)


#!/usr/bin/python
# coding=utf-8
import platform

from selenium import webdriver

from com_zxl_common import LogUtils

class BaseRequest:

    @staticmethod
    def get_web_content(url):
        LogUtils.i("get_web_content::" + url)
        #chromedriver = "C:\Program Files (x86)\Google\Chrome\Application\chromedriver.exe"
        chrome_driver = "/Users/zxl/Downloads/chromedriver"
        sys_str = platform.system()
        LogUtils.i("get_web_content::" + sys_str)
        if sys_str == 'Darwin':
            chrome_driver = "/Users/zxl/Downloads/chromedriver"
        elif sys_str == 'Windows':
            chrome_driver = "D:\\my_github_workspace\\chromedriver.exe"
        elif sys_str == 'Linux':
            # chrome_driver = "/Users/zxl/Downloads/chromedriver"
            chrome_driver = "/home/mi/下载/chromedriver"

        # 创建chrome参数对象
        opt = webdriver.ChromeOptions()

        opt.add_experimental_option('excludeSwitches', ['enable-automation'])

        # 把chrome设置成无界面模式，不论windows还是linux都可以，自动适配对应参数
        # opt.set_headless()
        opt.add_argument('--headless')
        # opt.add_argument('--proxy-server=http://127.0.0.1:8080')

        pre_fs = {"profile.managed_default_content_settings.images": 2}
        opt.add_experimental_option("prefs", pre_fs)

        # 创建chrome无界面对象
        driver = webdriver.Chrome(executable_path=chrome_driver, options=opt)

        driver.maximize_window()

        driver.get(url)

        return driver


from PIL import Image
import numpy as np
from IPython.display import display
import pickle
from sklearn.neighbors import KNeighborsClassifier

basedir_data = "./data/"
rel_path = basedir_data + "cifar-10-batches-py/"

def unpickle(file):
    with open(file, "rb") as fo:
        dict = pickle.load(fo, encoding="bytes")
    return dict

def default_label_fn(i, original):
    return original

def show_img(img_arr, label_arr, meta, index, label_fn=default_label_fn):
    """
    Given a numpy array of image from CIFAR-10 labels this method transform the data so that PIL can read and show the image.
    Check here how CIFAR encodes the image http://www.cs.toronto.edu/~kriz/cifar.html
    """
    one_img = img_arr[index, :]
    # Assume image size is 32 x 32. First 1024 px is r, next 1024 px is g, last 1024 px is b from the (r,g b) channel
    r = one_img[:1024].reshape(32, 32)
    g = one_img[1024:2048].reshape(32, 32)
    b = one_img[2048:].reshape(32, 32)
    rgb = np.dstack([r, g, b])
    img = Image.fromarray(np.array(rgb), "RGB")
    display(img)
    print(label_fn(index, meta[label_arr[index][0]].decode("utf-8")))

X = unpickle(rel_path + "data_batch_1")
img_data = X[b"data"]
img_label_orig = img_label = X[b"labels"]
img_label = np.array(img_label).reshape(-1, 1)

print(img_data)
print("shape", img_data.shape)

test_X = unpickle(rel_path + "test_batch")
test_data = test_X[b"data"]
test_label = test_X[b"labels"]
test_label = np.array(test_label).reshape(-1, 1)

sample_img_data = img_data[0:10, :]
print(sample_img_data)
print("shape", sample_img_data.shape)

batch = unpickle(rel_path + "batches.meta")
meta = batch[b"label_names"]
print(meta)

for i in range(0, 10):
    show_img(sample_img_data, img_label, meta, i)

def pred_label_fn(i, original):
    return original + "::" + meta[YPred[i]].decode("utf-8")

data_point_no = 10
sample_test_data = test_data[:data_point_no, :]

nbrs = KNeighborsClassifier(n_neighbors=3, algorithm="brute").fit(
    img_data, img_label_orig
)
YPred = nbrs.predict(sample_test_data)

for i in range(0, len(YPred)):
    show_img(sample_test_data, test_label, meta, i, label_fn=pred_label_fn)


from model import *
import numpy as np
import csv
import pandas as pd
import torch
import torchvision as ptv
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from model import Net

def load_data(dataname, labelname):
	data = pd.read_csv(dataname)
	label = pd.read_csv(dataname)
	print (data.shape)
	data = data.fillna(0)
	label = data.fillna(0)
	data = data.values
	label = label.values
	data1 = data[0:1000]
	label1 = label[0:1000]
	label1 = label1[:, 0]
	data1 = np.array(data1, dtype = np.float64)
	label1 = np.array(label1, dtype = np.float64)
	tensor = torch.from_numpy(data1)
	label_tensor = torch.from_numpy(label1)
	tensor = tensor.type(torch.FloatTensor)
	label_tensor = label_tensor.type(torch.FloatTensor)
	label_tensor = label_tensor.view(-1, 1)
	return tensor, label_tensor
if __name__ == '__main__':
	dataname = 'data.csv'
	labelname = 'label.csv'
	data, label = load_data(dataname, labelname)
	net = Net(input_dim = 54, hidden1 = 108, hidden2 = 108, hidden_pred = 27, output_dim = 1)
	encoder = Encoder(factor_dim = 27,  hidden_dim = 10, time_step = 5, num_layer = 3)
	# optimizer = torch.optim.SGD([{'params':net.parameters()},{'params':encoder.parameters()}], lr=0.2, momentum = 0.9)
	optimizer = torch.optim.SGD(net.parameters(), lr = 0.2, momentum = 0.9)
	loss_func = torch.nn.MSELoss()
	encoder = Encoder(factor_dim = 27,  hidden_dim = 10, time_step = 5, num_layer = 3)
	# attention = Attention(hidden_dim = 10, encoder_dim = 10)
	epoch = 500

	for i in range(epoch):
		print (i)
		prediction = net(data)
		# deep_factor = net(data)
		# pack_factor = deep_factor.view(4, 250, 27)
		# label = label.view(4, 250, 1)
		# prediction = encoder(pack_factor)
		# num_layer, batch, hidden_dim = prediction.shape
		# for i in range(2):
		# 	print (i)
		# 	weight = attention(out[:, i, :], hn[:, i, :], cn[:, i, :])
		# pack_factor = deep_factor.permute(1, 0, 2)

		loss = loss_func(prediction, label)
		optimizer.zero_grad()
		loss.backward()
		optimizer.step()
		print (loss)

		# if i % 5 == 0:
		# 	plot(data, label, prediction)


# -*- coding: utf-8 -*-
#                     The LLVM Compiler Infrastructure
#
# This file is distributed under the University of Illinois Open Source
# License. See LICENSE.TXT for details.

""" This module is responsible to capture the compiler invocation of any
build process. The result of that should be a compilation database.

This implementation is using the LD_PRELOAD or DYLD_INSERT_LIBRARIES
mechanisms provided by the dynamic linker. The related library is implemented
in C language and can be found under 'libear' directory.

The 'libear' library is capturing all child process creation and logging the
relevant information about it into separate files in a specified directory.
The input of the library is therefore the output directory which is passed
as an environment variable.

This module implements the build command execution with the 'libear' library
and the post-processing of the output files, which will condensates into a
(might be empty) compilation database. """

import logging
import subprocess
import argparse
import json
import sys
import os
import os.path
import re
import glob
import shlex
import pkg_resources
from analyzer import create_parser
from analyzer.decorators import to_logging_level, trace, entry
from analyzer.command import parse as cmdparse
from analyzer.command import Action

if 'darwin' == sys.platform:
    ENVIRONMENTS = [("ENV_OUTPUT", "BEAR_OUTPUT"),
                    ("ENV_PRELOAD", "DYLD_INSERT_LIBRARIES"),
                    ("ENV_FLAT", "DYLD_FORCE_FLAT_NAMESPACE")]
else:
    ENVIRONMENTS = [("ENV_OUTPUT", "BEAR_OUTPUT"),
                    ("ENV_PRELOAD", "LD_PRELOAD")]

@entry
def bear():
    """ Entry point for 'bear'.

        This part initializes some parts and forwards to the main method. """

    parser = initialize_command_line(create_parser())
    advanced = parser.add_argument_group('advanced options')
    advanced.add_argument(
        '--append',
        action='store_true',
        help="""Append new entries to existing compilation database.""")
    advanced.add_argument(
        '--disable-filter', '-n',
        dest='filtering',
        action='store_true',
        help="""Disable filter, unformated output.""")

    args = parser.parse_args()

    logging.getLogger().setLevel(to_logging_level(args.verbose))
    logging.debug(args)

    if args.help or 0 == len(args.build):
        parser.print_help()
        return 0

    return main(args)

def main(args):
    """ The reusable entry point of 'bear'.

        The 'scan-build' and 'bear' are the two entry points of this code.
        Both provide the parsed argument object as input for this job. """

    def load_current_cdb(filename):
        """ Load existing cdb elements when cdb file is readable. """
        if os.path.exists(filename):
            with open(filename) as handle:
                return json.load(handle)
        else:
            return []

    exit_code = 0
    with TemporaryDirectory(prefix='bear-') as tmpdir:
        exit_code = run_build(args.build, tmpdir)
        append = 'append' in args and args.append
        currents = load_current_cdb(args.cdb) if append else []
        filtering = 'filtering' in args and args.filtering
        commands = merge(currents, collect(not filtering, tmpdir))
        with open(args.cdb, 'w+') as handle:
            json.dump(commands, handle, sort_keys=True, indent=4)
    return exit_code

@trace
def initialize_command_line(parser):
    """ Add task related argument to the command line parser. """
    parser.add_argument(
        dest='build',
        nargs=argparse.REMAINDER,
        help="""Command to run.""")

    return parser

@trace
def run_build(command, destination):
    """ Runs the original build command.

    It sets the required environment variables and execute the given command.
    The exec calls will be logged by the 'libear' preloaded library. """

    def get_ear_so_file():
        lib_name = 'libear.dylib' if 'darwin' == sys.platform else 'libear.so'
        return pkg_resources.resource_filename('analyzer', lib_name)

    environment = dict(os.environ)
    for alias, key in ENVIRONMENTS:
        value = '1'
        if alias == 'ENV_PRELOAD':
            value = get_ear_so_file()
        elif alias == 'ENV_OUTPUT':
            value = destination
        environment.update({key: value})

    child = subprocess.Popen(command, env=environment)
    child.wait()
    return child.returncode

@trace
def collect(filtering, destination):
    """ Collect the execution information from the output directory. """
    def parse(filename):
        """ Parse the file generated by the 'libear' preloaded library. """
        RS = chr(0x1e)
        US = chr(0x1f)
        with open(filename, 'r') as handler:
            content = handler.read()
            records = content.split(RS)
            return {'pid': records[0],
                    'ppid': records[1],
                    'function': records[2],
                    'directory': records[3],
                    'command': records[4].split(US)[:-1]}

    def general_filter(iterator):
        """ Filter out the non compiler invocations. """
        def known_compiler(command):
            patterns = [
                re.compile(r'^([^/]*/)*c(c|\+\+)$'),
                re.compile(r'^([^/]*/)*([^-]*-)*g(cc|\+\+)(-[2345].[0-9])?$'),
                re.compile(r'^([^/]*/)*([^-]*-)*clang(\+\+)?(-[23].[0-9])?$'),
                re.compile(r'^([^/]*/)*llvm-g(cc|\+\+)$'),
            ]
            executable = command[0]
            for pattern in patterns:
                if pattern.match(executable):
                    return True
            return False

        def cancel_parameter(command):
            patterns = [
                re.compile(r'^-cc1$')
            ]
            for pattern in patterns:
                for arg in command[1:]:
                    if pattern.match(arg):
                        return True
            return False

        for record in iterator:
            command = record['command']
            if known_compiler(command) and not cancel_parameter(command):
                yield record

    def format_record(iterator):
        """ Generate the desired fields for compilation database entries. """
        def join_command(args):
            """ Create a single string from list.

            The major challenge, which is not solved yet, to deal with white
            spaces. Which are used by the shell as separator.
            (Eg.: -D_KEY="Value with spaces") """
            return ' '.join(args)

        for record in iterator:
            atoms = cmdparse({'command': record['command']}, lambda x: x)
            if atoms['action'] == Action.Compile:
                for filename in atoms['files']:
                    yield {'directory': record['directory'],
                           'command': join_command(record['command']),
                           'file': os.path.abspath(filename)}

    chain = lambda x: format_record(general_filter(x))

    generator = [parse(record)
                 for record
                 in glob.iglob(os.path.join(destination, 'cmd.*'))]
    return list(chain(generator)) if filtering else generator

@trace
def merge(old, new):
    """ Merge two list of commands into one. """
    def duplicate(state, entry):
        """ Find out repetition amongst the merged items. """
        def hash_cdb(entry):
            """ Make a unique hash for cdb entries to detect duplicates. """
            # On OS X the 'cc' and 'c++' compilers are wrappers for 'clang'
            # therefore both call would be logged. To avoid this the hash does
            # not contain the first word of the command.
            command = ' '.join(shlex.split(entry['command'])[1:])
            # For faster lookup in set filename is reverted
            filename = entry['file'][::-1]
            # For faster lookup in set directory is reverted
            directory = entry['directory'][::-1]
            return '<>'.join([filename, directory, command])

        if os.path.exists(entry['file']):
            entry_hash = hash_cdb(entry)
            if entry_hash not in state:
                state.add(entry_hash)
                return False
        return True

    state = set()
    return [entry for entry in old + new if not duplicate(state, entry)]

if sys.version_info.major >= 3 and sys.version_info.minor >= 2:
    from tempfile import TemporaryDirectory
else:
    class TemporaryDirectory(object):
        """ This function creates a temporary directory using mkdtemp() (the
        supplied arguments are passed directly to the underlying function).
        The resulting object can be used as a context manager. On completion
        of the context or destruction of the temporary directory object the
        newly created temporary directory and all its contents are removed
        from the filesystem. """
        def __init__(self, **kwargs):
            from tempfile import mkdtemp
            self.name = mkdtemp(*kwargs)

        def __enter__(self):
            return self.name

        def __exit__(self, _type, _value, _traceback):
            self.cleanup()

        def cleanup(self):
            from shutil import rmtree
            if self.name is not None:
                rmtree(self.name)


# Import Library
import numpy as np
import sklearn
import scipy
import pandas
from sklearn.svm import SVC
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.tree import DecisionTreeClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.model_selection import KFold, RepeatedKFold
from sklearn import metrics
from sklearn.model_selection import cross_val_score, cross_val_predict
from sklearn.metrics import (brier_score_loss, precision_score, recall_score, f1_score)
from sklearn.metrics import precision_recall_fscore_support
from sklearn.multiclass import OneVsOneClassifier
from sklearn.multiclass import OutputCodeClassifier
from sklearn.multiclass import OneVsRestClassifier
from sklearn.svm import LinearSVC
from sklearn.neighbors import KNeighborsClassifier

# Read data from csv file #
data = pandas.read_csv("data.csv");
# print(data.head());

# Divide data into training data and ground truth #
X = data.drop('QN', axis=1).drop('ground_truth', axis=1).drop('ground_truth_binary', axis=1);
y2 = data['ground_truth']; # full categorical ground truth
y = data['ground_truth_binary']; # binary ground truth
# print(X.head());
# print(Y);ß
# print(Y2);

##########
# K-Fold #
##########
########################################################################################################################
repeated_times = 10;
kf = RepeatedKFold(n_splits=5, n_repeats=repeated_times, random_state=np.random);
kf.get_n_splits(X);
# print(kf);

# Perform 5-fold cross validation #
svm_score = [0, 0, 0]; # single class score for svm
svm_score2 = [0, 0, 0]; # multi-class score for svm
gnb_score = [0, 0, 0]; # single class score for gnb
gnb_score2 = [0, 0, 0]; # multi-class score for gnb
knn_score = [0, 0, 0]; # single class score for knn
knn_score2 = [0, 0, 0]; # multi-class score for knn
dtree_score = [0, 0, 0]; # single class score for dtree
dtree_score2 = [0, 0, 0]; # multi-class score for dtree

for train_index, test_index in kf.split(X):
    split_training_sets = "TRAIN:" + str(train_index) + "\n" + "TEST:" + str(test_index);
    # print(split_training_sets);

    ##################################
    # Split into train and test sets #
    ##################################
    X_train, X_test = np.array(X)[train_index], np.array(X)[test_index];
    y_train, y_test = np.array(y)[train_index], np.array(y)[test_index];
    X2_train, X2_test = np.array(X)[train_index], np.array(X)[test_index];
    y2_train, y2_test = np.array(y2)[train_index], np.array(y2)[test_index];

    #######
    # SVM #
    #######
    svm = SVC(kernel='rbf', gamma=0.05, C=1);
    svm.fit(X_train, y_train);
    y_pred_svm = svm.predict(X_test);
    # svm2 = SVC(kernel='rbf', gamma=0.1, C=1);
    svm2 = OneVsRestClassifier(LinearSVC(random_state=0));
    svm2.fit(X2_train, y2_train);
    y2_pred_svm = svm2.predict(X2_test);
    # print(confusion_matrix(y_test, y_pred_svm));
    # print(classification_report(y_test, y_pred_svm));
    # print(svm.score(X_test, y_test));  # accuracy
    # print("\tF1: %1.3f\n" % f1_score(y_test, y_pred_svm));
    # print("\tLOSS: %1.3f\n" % brier_score_loss(y_test, y_pred_svm));
    # print("\tPrecision: %1.3f\n" % precision_score(y_test, y_pred_svm));
    # print("\tRecall: %1.3f\n" % recall_score(y_test, y_pred_svm));

    # average=None # include all information of classification report;
    # 'micro' compute the f1-score using the global count of true positives / false negatives, etc.
    # 'macro' makes the average of the f1-score for each class: that's the avg / total result above
    # 'weighted' computes a weighted average of the f1-score. the more elements a class has,
    # the more important the f1-score for this class in the computation.
    score_svm = precision_recall_fscore_support(y_test, y_pred_svm, beta=1.0, labels=None, pos_label=1, average='macro',
                                            warn_for=('precision', 'recall', 'f-score'), sample_weight=None);
    report_svm = classification_report(y_test,y_pred_svm);
    # print(score);
    score2_svm = precision_recall_fscore_support(y2_test, y2_pred_svm, beta=1.0, average='macro', pos_label=None,
                                             warn_for=('precision', 'recall', 'f-score'), sample_weight=None);
    report2_svm = classification_report(y2_test,y2_pred_svm);
    # print(score2);

    # Final score of svm calculate
    svm_score[0] = svm_score[0] + score_svm[0];
    svm_score[1] = svm_score[1] + score_svm[1];
    svm_score[2] = svm_score[2] + score_svm[2];
    svm_score2[0] = svm_score2[0] + score2_svm[0];
    svm_score2[1] = svm_score2[1] + score2_svm[1];
    svm_score2[2] = svm_score2[2] + score2_svm[2];

    # Store scores to database

    ####################################################################################################################
    ###############
    # Naive Bayes #
    ###############
    gnb = GaussianNB().fit(X_train, y_train);
    # Testfrom sklearn.metrics import classification_report, confusion_matrix
    y_pred_gnb = gnb.predict(X_test);

    # Report #
    # print(confusion_matrix(y_test, y_pred_gnb))
    # print(classification_report(y_test, y_pred_gnb))
    # print(gnb.score(X_test, y_test));  # accuracy

    # Naive Bayes multi-class
    gnb2 = GaussianNB().fit(X2_train, y2_train);
    # Testfrom sklearn.metrics import classification_report, confusion_matrix
    y2_pred_gnb = gnb2.predict(X2_test);

    # Score
    score_gnb = precision_recall_fscore_support(y_test, y_pred_gnb, beta=1.0, labels=None, pos_label=1,
                                                average='macro',
                                                warn_for=('precision', 'recall', 'f-score'), sample_weight=None);
    report_gnb = classification_report(y_test, y_pred_gnb);
    # print(score);
    score2_gnb = precision_recall_fscore_support(y2_test, y2_pred_gnb, beta=1.0, average='macro', pos_label=None,
                                                 warn_for=('precision', 'recall', 'f-score'), sample_weight=None);
    report2_gnb = classification_report(y2_test, y2_pred_gnb);

    # Final score of gnb calculate
    gnb_score[0] = gnb_score[0] + score_gnb[0];
    gnb_score[1] = gnb_score[1] + score_gnb[1];
    gnb_score[2] = gnb_score[2] + score_gnb[2];
    gnb_score2[0] = gnb_score2[0] + score2_gnb[0];
    gnb_score2[1] = gnb_score2[1] + score2_gnb[1];
    gnb_score2[2] = gnb_score2[2] + score2_gnb[2];
    ####################################################################################################################
    ###########################
    # KNN k-nearest neighbours#
    ###########################
    knn = KNeighborsClassifier(n_neighbors=7).fit(X_train, y_train);

    # Testfrom sklearn.metrics import classification_report, confusion_matrix
    y_pred_knn = knn.predict(X_test);

    # KNN k-nearest neighbours multi-class
    knn2 = KNeighborsClassifier(n_neighbors=7).fit(X2_train, y2_train);

    # Testfrom sklearn.metrics import classification_report, confusion_matrix
    y2_pred_knn = knn2.predict(X2_test);

    # Score
    score_knn = precision_recall_fscore_support(y_test, y_pred_knn, beta=1.0, labels=None, pos_label=1,
                                                average='macro',
                                                warn_for=('precision', 'recall', 'f-score'), sample_weight=None);
    report_knn = classification_report(y_test, y_pred_knn);
    # print(score);
    score2_knn = precision_recall_fscore_support(y2_test, y2_pred_knn, beta=1.0, average='macro', pos_label=None,
                                                 warn_for=('precision', 'recall', 'f-score'), sample_weight=None);
    report2_knn = classification_report(y2_test, y2_pred_knn);

    # Final score of knn calculate
    knn_score[0] = knn_score[0] + score_knn[0];
    knn_score[1] = knn_score[1] + score_knn[1];
    knn_score[2] = knn_score[2] + score_knn[2];
    knn_score2[0] = knn_score2[0] + score2_knn[0];
    knn_score2[1] = knn_score2[1] + score2_knn[1];
    knn_score2[2] = knn_score2[2] + score2_knn[2];
    ####################################################################################################################
    #################
    # Decision Tree #
    #################
    dtree = DecisionTreeClassifier(max_depth=3).fit(X_train, y_train);

    # Testfrom sklearn.metrics import classification_report, confusion_matrix
    y_pred_dtree = dtree.predict(X_test);

    # Decision Tree multi-class
    dtree2 = DecisionTreeClassifier(max_depth=3).fit(X2_train, y2_train);

    # Testfrom sklearn.metrics import classification_report, confusion_matrix
    y2_pred_dtree = dtree2.predict(X2_test);

    # Score
    score_dtree = precision_recall_fscore_support(y_test, y_pred_dtree, beta=1.0, labels=None, pos_label=1,
                                                average='macro',
                                                warn_for=('precision', 'recall', 'f-score'), sample_weight=None);
    report_dtree = classification_report(y_test, y_pred_dtree);
    # print(score);
    score2_dtree = precision_recall_fscore_support(y2_test, y2_pred_dtree, beta=1.0, average='macro', pos_label=None,
                                                 warn_for=('precision', 'recall', 'f-score'), sample_weight=None);
    report2_dtree = classification_report(y2_test, y2_pred_dtree);

    # Final score of dtree calculate
    dtree_score[0] = dtree_score[0] + score_dtree[0];
    dtree_score[1] = dtree_score[1] + score_dtree[1];
    dtree_score[2] = dtree_score[2] + score_dtree[2];
    dtree_score2[0] = dtree_score2[0] + score2_dtree[0];
    dtree_score2[1] = dtree_score2[1] + score2_dtree[1];
    dtree_score2[2] = dtree_score2[2] + score2_dtree[2];
    ####################################################################################################################

print("\nSVM");
print("Binary Classifier:");
svm_score[0] = svm_score[0]/(5*repeated_times);
print("\tPrecision: %1.4f" % svm_score[0]);
svm_score[1] = svm_score[1]/(5*repeated_times);
print("\tRecall: %1.4f" % svm_score[1]);
svm_score[2] = svm_score[2]/(5*repeated_times);
print("\tF1-score: %1.4f" % svm_score[2]);
print("Multi-class Classifier:");
svm_score2[0] = svm_score2[0]/(5*repeated_times);
print("\tPrecision: %1.4f" % svm_score2[0]);
svm_score2[1] = svm_score2[1]/(5*repeated_times);
print("\tRecall: %1.4f" % svm_score2[1]);
svm_score2[2] = svm_score2[2]/(5*repeated_times);
print("\tF1-score: %1.4f" % svm_score2[2]);

print("\nNaive Bayes");
print("Binary Classifier:");
gnb_score[0] = gnb_score[0]/(5*repeated_times);
print("\tPrecision: %1.4f" % gnb_score[0]);
gnb_score[1] = gnb_score[1]/(5*repeated_times);
print("\tRecall: %1.4f" % gnb_score[1]);
gnb_score[2] = gnb_score[2]/(5*repeated_times);
print("\tF1-score: %1.4f" % gnb_score[2]);
print("Multi-class Classifier:");
gnb_score2[0] = gnb_score2[0]/(5*repeated_times);
print("\tPrecision: %1.4f" % gnb_score2[0]);
gnb_score2[1] = gnb_score2[1]/(5*repeated_times);
print("\tRecall: %1.4f" % gnb_score2[1]);
gnb_score2[2] = gnb_score2[2]/(5*repeated_times);
print("\tF1-score: %1.4f" % gnb_score2[2]);

print("\nK-Nearest Neighbours");
print("Binary Classifier:");
knn_score[0] = knn_score[0]/(5*repeated_times);
print("\tPrecision: %1.4f" % knn_score[0]);
knn_score[1] = knn_score[1]/(5*repeated_times);
print("\tRecall: %1.4f" % knn_score[1]);
knn_score[2] = knn_score[2]/(5*repeated_times);
print("\tF1-score: %1.4f" % knn_score[2]);
print("Multi-class Classifier:");
knn_score2[0] = knn_score2[0]/(5*repeated_times);
print("\tPrecision: %1.4f" % knn_score2[0]);
knn_score2[1] = knn_score2[1]/(5*repeated_times);
print("\tRecall: %1.4f" % knn_score2[1]);
knn_score2[2] = knn_score2[2]/(5*repeated_times);
print("\tF1-score: %1.4f" % knn_score2[2]);

print("\nDesicion Tree");
print("Binary Classifier:");
dtree_score[0] = dtree_score[0]/(5*repeated_times);
print("\tPrecision: %1.4f" % dtree_score[0]);
dtree_score[1] = dtree_score[1]/(5*repeated_times);
print("\tRecall: %1.4f" % dtree_score[1]);
dtree_score[2] = dtree_score[2]/(5*repeated_times);
print("\tF1-score: %1.4f" % dtree_score[2]);
print("Multi-class Classifier:");
dtree_score2[0] = dtree_score2[0]/(5*repeated_times);
print("\tPrecision: %1.4f" % dtree_score2[0]);
dtree_score2[1] = dtree_score2[1]/(5*repeated_times);
print("\tRecall: %1.4f" % dtree_score2[1]);
dtree_score2[2] = dtree_score2[2]/(5*repeated_times);
print("\tF1-score: %1.4f" % dtree_score2[2]);

########################################################################################################################
# End of K-Fold #
########################################################################################################################

Experiment/5 SVM KNN .../kfold_svm_bayes_cnn_macro.py


import logging
import os

import requests
from google.cloud import datastore

logger = logging.getLogger(__name__)

def broadcast(check_result):
    notification = _create_notification_from_check_result(check_result)
    for token in _get_tokens():
        _send_notification(token, notification)

def _create_notification_from_check_result(check_result):
    title = 'In stock!' if check_result.in_stock else 'Out of stock...'
    return {
        'title': '[{}] {}'.format(check_result.store_key, title),
        'body': check_result.name,
        'click_action': check_result.url,
    }

def _get_tokens():
    client = datastore.Client()
    for token in client.query(kind='Token').fetch():
        yield token['token_id']

def _send_notification(to, notification):
    server_key = os.environ['X2SWITCH_FIREBASE_SERVER_KEY']
    headers = {
        'Authorization': 'key={}'.format(server_key),
    }
    message = {
        'to': to,
        'notification': notification,
    }
    response = requests.post('https://fcm.googleapis.com/fcm/send',
                             json=message, headers=headers)
    response.raise_for_status()
    logger.info('Notification result: %d %r',
                response.status_code, response.json())


# experiment result wrapper

import numpy as np

import uuid
import h5py

from git import Repo
from pymongo import MongoClient

path_to_repo = '~/Documents/Thesis/latent_ssvm'
path_to_datafile = '/home/dmitry/Documents/Thesis/latent_ssvm/notebooks/experiment_data.hdf5'

class experiment(object):
    def __init__(self, f):
        self.f = f

    def __call__(self, description, *args, **kwargs):
        result = None
        try:
            result = self.f(*args, **kwargs)
        except:
            raise
        result.save(description)
        return result

class ExperimentResult(object):
    def __init__(self, data, meta, is_new=True):
        # meta information, comments, parameters
        # this will be stored in mongodb
        self.meta = meta
        if is_new:
            repo = Repo(path_to_repo)
            self.meta['commit_hash'] = repo.head.commit.hexsha
            # unique experiment identifier
            self.meta['id'] = uuid.uuid1().hex
        # generated data like scores per iteration, model parameters
        # stored in hdf5 file
        self.data = data

    def save_data(self):
        f = h5py.File(path_to_datafile, 'a', libver='latest')
        grp = f[self.meta['dataset_name']].create_group(self.meta['id'])
        for k in self.data.keys():
            grp.create_dataset(k, data=self.data[k])
        f.close()
        return grp.id.id

    def save_meta(self):
        client = MongoClient()
        client['lSSVM']['base'].insert(self.meta)
        client.disconnect()

    def save(self, description=''):
        self.description = description
        self.save_meta()
        self.save_data()
        return self.id

    @staticmethod
    def load(exp_id):
        client = MongoClient()
        meta = client['lSSVM']['base'].find_one({'id' : exp_id})
        f = h5py.File(path_to_datafile, 'r', libver='latest')
        grp = f[meta[u'dataset_name']][exp_id]
        data = {}
        for k in grp.keys():
            data[k] = np.empty(grp[k].shape)
            grp[k].read_direct(data[k])
        f.close()
        return ExperimentResult(data, meta, is_new=False)

    @property
    def description(self):
        return self.meta['description']

    @description.setter
    def description(self, description_):
        self.meta['description'] = description_

    @property
    def id(self):
        return self.meta['id']


import numpy as np
from shapely.geometry import Point, Polygon, LineString
import time

from .base import Trimesh

from copy import deepcopy

from scipy.spatial import cKDTree
from collections import deque
import logging
log = logging.getLogger(__name__)
log.addHandler(logging.NullHandler())

try:
    import occmodel as occ
    from geotools import Transform
except:
    log.error('Failed to import occmodel!')

def solid_to_mesh(solid):
    occ_mesh = solid.createMesh()
    occ_mesh.optimize()
    faces    = np.array(list(occ_mesh.triangles)).reshape((-1,3)).astype(int)
    vertices = np.array(list(occ_mesh.vertices)).reshape((-1,3)).astype(float)
    mesh     = Trimesh(vertices=vertices, faces=faces, process=True)
    if not mesh.is_watertight():
        #raise NameError('Mesh returned from openCASCADE isn\'t watertight!')
        log.error('Mesh returned from openCASCADE isn\'t watertight!')
    return mesh

def three_dimensionalize(points):
    shape = np.shape(points)
    if len(shape) != 2: raise NameError('Points must be 2D array!')
    if shape[1] == 2:
        points = np.column_stack((points, np.zeros(len(points))))
    elif shape[1] != 3: raise NameError('Points must be (n,2) or (n,3)!')
    return points

def extrude_shapely(polygon,
                    height,
                    height_pad   = [0,0],
                    contour_pad  = 0,
                    return_solid = False):

    shell = np.array(polygon.exterior.coords)
    holes = [np.array(i.coords) for i in polygon.interiors]
    mesh  = extrude_points(shell        = shell,
                           holes        = holes,
                           height       = height,
                           height_pad   = height_pad,
                           return_solid = return_solid)
    return mesh

def extrude_points(shell,
                   vectors    = None,
                   height     = None,
                   holes      = None,
                   return_solid = False,
                   height_pad = [0,0]):

    if vectors is None:
        pad_signed = np.sign(height) * np.array(height_pad)
        vectors    = np.array([[0.0, 0.0, -pad_signed[0]],
                               [0.0, 0.0,  pad_signed[1] + height]])
    else:
        vectors = np.array(vectors)

    shell_3D      = three_dimensionalize(shell)
    shell_3D[:,2] = vectors[0][2]

    shell_face  = occ.Face().createPolygonal(shell_3D)
    shell_solid = occ.Solid().extrude(shell_face, *vectors)

    if holes is None:
        if return_solid: return shell_solid
        else:            return solid_to_mesh(shell_solid)

    pad_vector  = np.diff(vectors, axis=0)[0]
    cut_vectors = vectors + [-pad_vector, pad_vector]

    for hole in holes:
        cut_face  = occ.Face().createPolygonal(three_dimensionalize(hole))
        cut_solid = occ.Solid().extrude(cut_face, *cut_vectors)
        shell_solid.cut(cut_solid)

    if return_solid: return shell_solid
    else:            return solid_to_mesh(shell_solid)

def remove_close(points, radius):
    '''
    Given an (n, m) set of points where n=(2|3) return a list points
    where no point is closer than radius
    '''
    tree     = cKDTree(points)
    consumed = np.zeros(len(points), dtype=np.bool)
    unique   = np.zeros(len(points), dtype=np.bool)
    for i in xrange(len(points)):
        if consumed[i]: continue
        neighbors = tree.query_ball_point(points[i], r=radius)
        consumed[neighbors] = True
        unique[i]           = True
    return points[unique]

def remove_close_set(points_fixed, points_reduce, radius):
    '''
    Given two sets of points and a radius, return a set of points
    that is the subset of points_reduce where no point is within
    radius of any point in points_fixed
    '''
    tree_fixed  = cKDTree(points_fixed)
    tree_reduce = cKDTree(points_reduce)
    reduce_duplicates = tree_fixed.query_ball_tree(tree_reduce, r = radius)
    reduce_duplicates = np.unique(np.hstack(reduce_duplicates).astype(int))
    reduce_mask = np.ones(len(points_reduce), dtype=np.bool)
    reduce_mask[reduce_duplicates] = False
    points_clean = points_reduce[reduce_mask]
    return points_clean

def sample_polygon(polygon, count=None):
    '''
    Randomly sample inside a polygon using rejection sampling,
    returning count number of points
    '''
    if count is None:
        count = len(polygon.exterior.coords) * 2
    bounds   = np.reshape(polygon.bounds, (2,2))
    box_size = np.ptp(bounds, axis=0)
    samples  = deque()
    while len(samples) < count:
        test_point = (np.random.random(2) * box_size) + bounds[0]
        if polygon.intersects(Point(test_point)):
            samples.append(test_point)
    return np.array(samples)

def nodes_random(polygon, radius=None, return_index=False):
    points_fixed = np.array(polygon.exterior.coords)
    polygon_idx  = [[0,len(points_fixed)]]
    if len(polygon.interiors) != 0:
        points_interior = np.vstack([i.coords for i in polygon.interiors])
        points_fixed    = np.vstack((points_fixed, points_interior))

    if radius is None:
        radius = np.ptp(np.reshape(polygon.bounds, (2,2)),axis=0).min() / 15.0

    # sample random points from inside the polygon with re
    points_random = sample_polygon(polygon)
    points_random = remove_close(points_random, radius)
    points_random = remove_close_set(points_fixed, points_random, radius)

    nodes = np.vstack((points_fixed, points_random))
    return nodes

def delaunay_polygon(polygon, points=None):
    '''
    Calculates Delaunay trianglation of a polygon,
    discarding triangles which are not inside the polygon

    If points is not specified, it is calculated as the exterior of
    the polygon and a number of points randomly sampled from
    inside the polygon
    '''
    if points is None:
        points = nodes_random(polygon)
    faces       = Delaunay(points).simplices.copy()
    test_points = points[faces].mean(axis=1)
    intersects  = np.array([polygon.intersects(Point(i)) for i in test_points])
    faces       = faces[intersects]
    return points, faces

if __name__ == '__main__':
    formatter = logging.Formatter("[%(asctime)s] %(levelname)-7s (%(filename)s:%(lineno)3s) %(message)s", "%Y-%m-%d %H:%M:%S")
    handler_stream = logging.StreamHandler()
    handler_stream.setFormatter(formatter)
    level = logging.DEBUG
    handler_stream.setLevel(level)
    for logger in [log, logging.getLogger('trimesh')]:
        logger.addHandler(handler_stream)
        logger.setLevel(level)

    from scipy.spatial import Delaunay
    from shapely.geometry import MultiPoint
    polygon = Point([0,0]).buffer(1).union(Point([.95,0]).buffer(1)).difference(Point([0,0]).buffer(.5))

    points, faces = delaunay_polygon(polygon)

    import matplotlib.pyplot as plt
    plt.plot(*polygon.exterior.xy)
    plt.triplot(*points.T, triangles=faces)
    plt.show()

    a = Point([0,0]).buffer(1)
    m = extrude_shapely(a, height=1, height_pad=[-.1,.1])


r"""
===============================================================================
Submodule -- throat_diameter
===============================================================================

"""
from . import misc as _misc
import scipy as _sp

def weibull(geometry, shape, scale, loc, seeds=None, **kwargs):
    if seeds not in geometry:
        geometry['throat.seed'] = _sp.rand(geometry.Nt,)
    return _misc.weibull(geometry=geometry, shape=shape, scale=scale, loc=loc,
                         seeds=seeds)
weibull.__doc__ = _misc.weibull.__doc__

def normal(geometry, scale, loc, seeds=None, **kwargs):
    if seeds not in geometry:
        geometry['throat.seed'] = _sp.rand(geometry.Nt,)
    return _misc.normal(geometry=geometry, scale=scale, loc=loc,
                        seeds=seeds)
normal.__doc__ = _misc.normal.__doc__

def generic(geometry, func, seeds=None, **kwargs):
    if seeds not in geometry:
        geometry['throat.seed'] = _sp.rand(geometry.Nt,)
    return _misc.generic(geometry=geometry, func=func, seeds=seeds)
generic.__doc__ = _misc.generic.__doc__

def random(geometry, seed=None, num_range=[0, 1], **kwargs):
    r"""
    Assign throat sizes from a random

    Parameters
    ----------
    geometry : OpenPNM Geometry object
        The Geometry object to which this model will apply.  This is necessary
        to determine the length of the array to generate.

    seed : int
        The starting seed value to send to Scipy's random number generator.
        The default is None, which means different distribution is returned
        each time the model is run.

    num_range : list
        A two element list indicating the low and high end of the returned
        numbers.  The default is [0, 1] but this can be adjusted to produce
        throat sizes directly; for instance pores between 10 and 100 um can be
        generated with ``num_range = [0.00001, 0.0001]``.
    """
    return _misc.random(N=geometry.Nt, seed=seed, num_range=num_range)

def cylinder(geometry, tsd_name, tsd_shape, tsd_loc, tsd_scale,
             throat_seed='throat.seed', tsd_offset=0, **kwargs):
    r"""
    Calculate throat diameter from seeds for a cylindrical throat

    Parameters
    ----------
    geometry : OpenPNM Geometry Object
        The Geometry object which this model is associated with. This controls
        the length of the calculated array, and also provides access to other
        necessary geometric properties.

    tsd_name : string
        The name of the statistical distribution to use. This model uses the
        Scipy.stats module, so any of the distributions available there are
        suitable options.

    tsd_shape, loc and scale : scalars
        The parameters to send to the selected statistics model.  Most of the
        Scipy.stats models accept the same keyword arguments.  Note that the
        psd_ prefix is added by OpenPNM to indicate 'pore size distribution'.

    tsd_offset : scalar
        Controls the minimum value in the pore size distribution by shifting
        the entire set of values by the given offset.  This is useful for
        avoiding pore sizes too close to zero.

    Notes
    -----
    This pore-scale model is deprecated.  Use ``weibull``, ``normal`` or
    ``generic`` to get produce pore sizes distributions.
    """
    import scipy.stats as spst
    prob_fn = getattr(spst, tsd_name)
    P = prob_fn(tsd_shape, loc=tsd_loc, scale=tsd_scale)
    value = P.ppf(geometry[throat_seed]) + tsd_offset
    return value

def equivalent_circle(geometry, throat_area='throat.area', **kwargs):
    r"""
    Calculates the diameter of a cirlce with same area as the throat.

    Parameters
    ----------
    geometry : OpenPNM Geometry object
        The Geometry object which contains the throat area values

    thorat_area : string
        The dictionary key to the throat area values
    """
    areas = geometry[throat_area]
    value = 2*_sp.sqrt(areas/_sp.pi)  # 64 bit sqrt doesn't work!
    return value

def minpore(network, geometry, factor=1, **kwargs):
    r"""
    Assign the throat diameter to be equal to the smallest connecting pore
    diameter. If zero (in case of boundaries) take it to be the maximum of
    the connecting pore diameters

    Parameters
    ----------
    factor : float < 1
        A factor between 0 and 1 to further constrict the throat size
        calculcated by the function.

    """
    gTs = geometry.throats()
    nTs = geometry.map_throats(network, gTs)
    pDs = network["pore.diameter"][network["throat.conns"][nTs]]
    value = _sp.amin(pDs, axis=1)*factor
    value[value == 0.0] = _sp.amax(pDs, axis=1)[value == 0.0]
    return value


import time, boto3, datetime, sys, csv, os, json

today = datetime.datetime.now().strftime("%Y%m%d")
year = datetime.datetime.now().strftime("%Y")
env = sys.argv[1] # input parameter

# open an AWS IAM boto session for the environment selected (live, test, security zone)
session = boto3.Session(profile_name=str(env))
iam_client = session.client("iam")

# open an AWS S3 boto session for the security tool instance
s3_client = boto3.client("s3")

# generate AWS IAM report (max 4 times per day)
try:
    iam_client.generate_credential_report()
except:
    pass

# wait for report generation
time.sleep(30)

# retrieve the report
report = iam_client.get_credential_report()

# output the results in a flat file
with open('/usr/local/src/security_scripts/outputs/'+today+'-aws-tmp'+ env +'-iam.csv', 'w+') as tmpFile:
    tmpFile.write(str(report['Content'],'utf-8'))

with open('/usr/local/src/security_scripts/outputs/' + today + '-aws-tmp' + env + '-iam.csv','r') as tmpFile, open('/usr/local/src/security_scripts/outputs/' + today + '-aws-' + env + '-iam.csv','w+') as outFile:
    lines = csv.reader(tmpFile)
    outFile.write("username,arn,user_creation_time,password_enabled,password_last_used,password_last_changed,password_next_rotation,mfa_active,access_key_1_active,access_key_1_last_rotated,access_key_1_last_used_date,access_key_1_last_used_region,access_key_1_last_used_service,access_key_2_active,access_key_2_last_rotated,access_key_2_last_used_date,access_key_2_last_used_region,access_key_2_last_used_service,cert_1_active,cert_1_last_rotated,cert_2_active,cert_2_last_rotated,groups\n")
    next(lines)
    for line in lines:
        # get all groups and policies attached for each user using AWS CLI as this is not provided into AWS credential report
        groups = ""
        #print(line[0])
        if line[0] != '<root_account>':
            attached_groups = iam_client.list_groups_for_user(UserName=line[0])['Groups']
            final_attached_groups = ''
            i = 0
            for group in attached_groups:
                if final_attached_groups != '' and attached_groups[i]['GroupName'] != '':
                    final_attached_groups = final_attached_groups + '|' + ''.join(attached_groups[i]['GroupName'])
                elif attached_groups[i]['GroupName'] != '':
                    final_attached_groups = ''.join(attached_groups[i]['GroupName'])
                i = i+1
            attached_managed_policies = iam_client.list_attached_user_policies(UserName = line[0])['AttachedPolicies']
            final_attached_managed_policies = ''
            i = 0
            for managed_policies in attached_managed_policies:
                if final_attached_managed_policies != '' and attached_managed_policies[i]['PolicyName'] != '':
                    final_attached_managed_policies = final_attached_managed_policies + '|' + ''.join(attached_managed_policies[i]['PolicyName'])
                elif attached_managed_policies[i]['PolicyName'] != '':
                    final_attached_managed_policies = ''.join(attached_managed_policies[i]['PolicyName'])
                i = i + 1
            attached_inline_policies = iam_client.list_user_policies(UserName = line[0])['PolicyNames']
            final_attached_inline_policies = ''.join(attached_inline_policies)
            groups = final_attached_groups + final_attached_inline_policies + final_attached_managed_policies
        #print(groups)
        outFile.write(line[0]+","+line[1]+","+line[2]+","+line[3]+","+line[4]+","+line[5]+","+line[6]+","+line[7]+","+line[8]+","+line[9]+","+line[10]+","+line[11]+","+line[12]+","+line[13]+","+line[14]+","+line[15]+","+line[16]+","+line[17]+","+line[18]+","+line[19]+","+line[20]+","+line[21]+","+groups+"\n")

# upload the result to s3
s3_client.upload_file('/usr/local/src/security_scripts/outputs/'+ today +'-aws-'+ env +'-iam.csv','securityzone-audit', 'aws-' + env + '-iam/' + year + '/' + today + '-aws-' + env + '-iam.csv')


"""新建一个组件."""
from typing import Dict, Any
from pmfp.const import (
    GOLBAL_PYTHON_VERSION
)
from pmfp.utils import (
    get_golang_version,
    get_node_version
)
from ._new_component import new_component
from ._new_doc import new_document
from ._new_env import new_env
from ._new_setup import new_setup
from ._new_readme import new_readme
from ._new_pb import new_pb
from ._new_es_script import new_es_script
from ._new_test import new_test

def new(config: Dict[str, Any], kwargs: Dict[str, Any]):
    """新建一个组件.

    Args:
        config (Dict[str, Any]): 项目配置
        kwargs (Dict[str, Any]): 组件配置

    Raises:
        e: 组件创建错误

    """
    if kwargs["language"] == "-":
        c_language = config["project-language"]
    else:
        c_language = kwargs["language"]

    if kwargs["to"] == "-":
        to = config["project-name"]
    else:
        to = kwargs["to"]
    c_name = kwargs["component_name"]
    if c_name in ("document", "doc"):
        new_document(config, c_language.lower())
    elif c_name == "env":
        new_env(config, c_language.lower())
    elif c_name == "readme":
        new_readme(config)
    elif c_name in ("setup", "cython_setup", "cython_numpy_setup", "cmd_setup"):
        new_setup(config, c_language, c_name)
    elif c_name in ("pb", "grpc", "grpc-streaming", "grpc-web-proxy"):
        if kwargs["rename"] == "-":
            rename = config["project-name"]
        elif kwargs["rename"] == "":
            rename = c_name
        else:
            rename = kwargs["rename"]
        new_pb(c_name=c_name, rename=rename, to=to, project_name=config["project-name"])
    elif c_name == "test":
        project_name = config["project-name"]
        if kwargs["rename"] == "-":
            rename = config["project-name"]
        elif kwargs["rename"] == "":
            rename = c_name
        else:
            rename = kwargs["rename"]
        new_test(c_language, project_name, rename)
    elif c_name == "es_script" and c_language == "Javascript":
        new_es_script(config)
    else:
        if not kwargs.get("kwargs"):
            kwargs["kwargs"] = {}
        if c_language == "Golang":
            kwargs["kwargs"]["language_version"] = get_golang_version() or "latest"
        elif c_language == "Python":
            kwargs["kwargs"]["language_version"] = GOLBAL_PYTHON_VERSION or "latest"
        elif c_language == "Javascript":
            kwargs["kwargs"]["language_version"] = get_node_version() or "latest"
        spl_name = c_name.split("-")
        c_category = spl_name[0]
        c_name = "".join(spl_name[1:])
        if kwargs["rename"] == "-":
            rename = config["project-name"]
        elif kwargs["rename"] == "":
            rename = c_name
        else:
            rename = kwargs["rename"]
        c_language = c_language.lower()
        c_path = f"{c_language}/{c_category}/{c_name}"
        test = kwargs["test"]
        try:
            nc_kwargs = kwargs.get("kwargs", {})
            print(f"创建组件{c_path},kwargs: {nc_kwargs}")
            new_component(
                config,
                path=c_path,
                to=to,
                rename=rename,
                test=test,
                **nc_kwargs)
        except Exception as e:
            print(f"组件{c_path}创建错误")
            raise e
        else:
            print(f"{c_path}创建为{rename}成功")


from django.shortcuts import render, redirect
from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import User
from django.http import HttpResponseRedirect, HttpResponse, Http404
from django.urls import reverse

from django.contrib import messages
import datetime

from django.db.models import Q
from .models import *

@login_required
def index(request):
    service_posts = ServicePost.objects.filter().order_by('-created_datetime')[:10]

    return render(request, 'service/index.html', {
        'navigation': 'service',
        'service_posts': service_posts,
        'unread_notifications_count': Notification.objects.filter(is_read=False,owner=request.user).count()
    })

@login_required
def explore(request):
    service_posts = ServicePost.objects.filter().order_by('-created_datetime')
    return render(request, 'service/explore.html', {
        'navigation': 'service',
        'service_posts': service_posts,
        'unread_notifications_count': Notification.objects.filter(is_read=False,owner=request.user).count()
    })

@login_required
def search(request):

    if request.method == 'GET':
        search_text = request.GET.get('service_search_text')

        title_matches = ServicePost.objects.filter(
            Q(title__icontains=search_text) | Q(service__name__icontains=search_text)
        )

        description_matches = ServicePost.objects.filter(
            Q(description__icontains=search_text) | Q(service__info__icontains=search_text)
        )

        location_matches = ServicePost.objects.filter(
            Q(service__location__icontains=search_text)
        )

        tag_matches = ServicePost.objects.filter(

        ).filter(service__tags__label__icontains=search_text)

    return render(request, 'service/search.html', {
        'search_text': search_text,
        'navigation': 'service',
        'title_matches': title_matches,
        'description_matches': description_matches,
        'location_matches': location_matches,
        'tag_matches': tag_matches,
        'unread_notifications_count': Notification.objects.filter(is_read=False,owner=request.user).count()
    })

@login_required
def create(request):
    if request.method == 'POST':
        # Creating new service and saving it
        new_service = Service()
        new_service.name = request.POST.get('service-name')
        new_service.info = request.POST.get('service-description')
        new_service.start_datetime = request.POST.get('service-start-datetime')
        new_service.end_datetime = request.POST.get('service-end-datetime')
        new_service.location = request.POST.get('service-location')

        new_service.save()
        tags = request.POST.getlist('service-tags')
        for tag_name in tags:
            tag, created = Tag.objects.get_or_create(
                label = tag_name
            )

            new_service.tags.add(tag)

        new_service.save()

        # Creating new service post record and saving it
        new_post = ServicePost()
        new_post.title = request.POST.get('post-title')
        new_post.description= request.POST.get('post-description')
        new_post.is_request_post = (request.POST['is_request_post'] == '1')
        new_post.owner = User.objects.get(username=request.user.username)
        new_post.created_datetime = datetime.datetime.now()
        new_post.service = new_service

        new_post.save()

        notification = Notification()
        notification.owner = request.user
        notification.created_datetime = datetime.datetime.now()
        notification.text = "You created a new service post titled '" + request.POST.get('post-title') + "'"
        notification.target_url = ""
        notification.save()

        messages.success(request, "Post Published!", extra_tags='message-head')

        messages.success(request, 'Hey ' + request.user.first_name + '! We have recorded and published your post!', extra_tags="message-body")

        return HttpResponseRedirect(reverse('service:create'))

    return render(request, 'service/create.html', {
        'navigation': 'service',
        'unread_notifications_count': Notification.objects.filter(is_read=False,owner=request.user).count()
    })

@login_required
def detail(request, post_id):
    try:
        post = ServicePost.objects.get(pk=post_id)
        comments = ServicePostComment.objects.filter(post=post).order_by('-created_datetime')

    except ServicePost.DoesNotExist:
        raise Http404('Post does not exist')

    return render(request, 'service/details.html', {
        'navigation': 'service',
        'post': post,
        'comments': comments,
        'unread_notifications_count': Notification.objects.filter(is_read=False,owner=request.user).count()
    })

@login_required
def update(request, post_id):
    try:
        post = ServicePost.objects.get(pk=post_id)
    except ServicePost.DoesNotExist:
        raise Http404('Post does not exist')

    if request.method == 'POST':

        post.service.name = request.POST.get('service-name')
        post.service.info = request.POST.get('service-description')
        post.service.start_datetime = request.POST.get('service-start-datetime')
        post.service.end_datetime = request.POST.get('service-end-datetime')
        post.service.location = request.POST.get('service-location')

        post.service.save()
        post.save()

        post.service.tags.clear()
        tags = request.POST.getlist('service-tags')
        for tag_name in tags:
            tag, created = Tag.objects.get_or_create(
                label = tag_name
            )

            post.service.tags.add(tag)

        post.service.save()
        post.save()

        post.title = request.POST.get('post-title')
        post.description= request.POST.get('post-description')
        post.is_request_post = (request.POST['is_request_post'] == '1')
        post.modified_datetime = datetime.datetime.now()

        post.save()

        notification = Notification()
        notification.owner = request.user
        notification.created_datetime = datetime.datetime.now()
        notification.text = "You updated your service post information. The post is titled '" + request.POST.get('post-title') + "'"
        notification.target_url = ""
        notification.save()

        messages.success(request, "Post Updated!", extra_tags='message-head')

        messages.success(request, 'Hey ' + request.user.first_name + '! We have updated your post!', extra_tags="message-body")

        return HttpResponseRedirect(reverse('service:update', kwargs={'post_id': post.id}))

    return render(request, 'service/update.html', {
        'navigation': 'service',
        'post': post,
        'unread_notifications_count': Notification.objects.filter(is_read=False,owner=request.user).count()
    })

@login_required
def close(request, post_id):
    try:
        post = ServicePost.objects.get(pk=post_id)
    except ServicePost.DoesNotExist:
        raise Http404('Post does not exist')

    if request.user != post.owner:
        messages.warning(request, "Unauthorized Attempt", extra_tags='message-head')
        messages.warning(request, "You're not authorized to perform this operation. Only the post owner can close the post.", extra_tags='message-body')

    else:
        post.closed_datetime = datetime.datetime.now()
        post.save()

        notification = Notification()
        notification.owner = request.user
        notification.created_datetime = datetime.datetime.now()
        notification.text = "You recently closed a service post titled '" + post.title + "'"
        notification.target_url = ""
        notification.save()

        messages.success(request, "Post closed", extra_tags='message-head info')
        messages.warning(request, "You've recently closed a service post. You will no require receive comments in that post, although the post is still visible unless you delete it!", extra_tags='message-body')

    return HttpResponseRedirect(reverse('service:detail', kwargs={'post_id': post.id}))

@login_required
def delete(request, post_id):
    try:
        post = ServicePost.objects.get(pk=post_id)
    except ServicePost.DoesNotExist:
        raise Http404('Post does not exist')

    if request.user != post.owner:
        messages.warning(request, "Unauthorized Attempt", extra_tags='message-head error')
        messages.warning(request, "You're not authorized to perform this operation. Only the post owner can delete the post.", extra_tags='message-body')

    else:
        post.service.delete()
        post.delete()

        notification = Notification()
        notification.owner = request.user
        notification.created_datetime = datetime.datetime.now()
        notification.text = "You deleted your service post titled '" + post.title + "'"
        notification.target_url = ""
        notification.save()

        messages.success(request, "Post Deleted", extra_tags='message-head info')
        messages.warning(request, "You've successfully deleted a service post. You will no longer be able to access the post!", extra_tags='message-body')

    return HttpResponseRedirect(reverse('service:index'))

@login_required
def create_comment(request, post_id):
    try:
        post = ServicePost.objects.get(pk=post_id)
    except ServicePost.DoesNotExist:
        raise Http404('Post does not exist')

    if request.method == 'POST':
        comment_text = request.POST.get('comment')

        new_comment = ServicePostComment()
        new_comment.text = comment_text
        new_comment.created_datetime = datetime.datetime.now()

        new_comment.owner = request.user
        new_comment.post = post

        new_comment.is_private = False
        if request.POST.get('is-private'):
            new_comment.is_private = True

        new_comment.save()

        notification = Notification()
        notification.owner = post.owner
        notification.created_datetime = datetime.datetime.now()
        notification.text = request.user.first_name + " " + request.user.last_name + " commented in your service post!"
        notification.target_url = ""
        notification.save()

        messages.success(request, "New Comment Added!", extra_tags='message-head info')
        messages.warning(request, "Your comment has been added to the post!", extra_tags='message-body')

    return HttpResponseRedirect(reverse('service:detail', kwargs={ 'post_id': post.id }))


import voluptuous as vol

import esphomeyaml.config_validation as cv
from esphomeyaml.components import binary_sensor
from esphomeyaml.components.pn532 import PN532Component
from esphomeyaml.const import CONF_NAME, CONF_UID
from esphomeyaml.core import HexInt
from esphomeyaml.helpers import ArrayInitializer, get_variable

DEPENDENCIES = ['pn532']

CONF_PN532_ID = 'pn532_id'

def validate_uid(value):
    value = cv.string_strict(value)
    for x in value.split('-'):
        if len(x) != 2:
            raise vol.Invalid("Each part (separated by '-') of the UID must be two characters "
                              "long.")
        try:
            x = int(x, 16)
        except ValueError:
            raise vol.Invalid("Valid characters for parts of a UID are 0123456789ABCDEF.")
        if x < 0 or x > 255:
            raise vol.Invalid("Valid values for UID parts (separated by '-') are 00 to FF")
    return value

PLATFORM_SCHEMA = cv.nameable(binary_sensor.BINARY_SENSOR_PLATFORM_SCHEMA.extend({
    vol.Required(CONF_UID): validate_uid,
    cv.GenerateID(CONF_PN532_ID): cv.use_variable_id(PN532Component)
}))

def to_code(config):
    hub = None
    for hub in get_variable(config[CONF_PN532_ID]):
        yield
    addr = [HexInt(int(x, 16)) for x in config[CONF_UID].split('-')]
    rhs = hub.make_tag(config[CONF_NAME], ArrayInitializer(*addr, multiline=False))
    binary_sensor.register_binary_sensor(rhs, config)


import numpy as np
import cv2
import matplotlib.pyplot as plt
from PIL import Image
from skimage.io import imread, imsave
#==============================================================================
test_f = open("E:/ML/all_data/all_data/test_images.txt", "r")
test_data = test_f.readlines()
test_f.close()

test_f2 = open("E:/ML/all_data/all_data/test_labels.txt", "r")
test_labels = test_f2.readlines()
test_f2.close()

train_f = open("E:/ML/all_data/all_data/train_images.txt", "r")
train_data = train_f.readlines()
train_f.close()

train_f2 = open("E:/ML/all_data/all_data/train_labels.txt", "r")
train_labels = train_f2.readlines()
train_f2.close()

test_data = test_data[0].split()
test_labels = test_labels[0].split()
train_data = train_data[0].split()
train_labels = train_labels[0].split()

temp_img = cv2.imread("E:/ML/all_data/all_data/img.png")
temp_img = cv2.cvtColor(temp_img, cv2.COLOR_BGR2GRAY)
temp_img = cv2.resize(temp_img, (28, 28))

data = train_data
labels = train_labels

arr = []
temp = []
cnt = 0

for i in range(0, len(data)):

    temp.append(int(data[i]))

    cnt+=1
    if cnt==784:
        arr.append(temp)
        temp = []
        cnt=0

cnt_ = 3600
for i in range(0, len(arr)):

    arr2 = arr[i]
    len1 = len(arr2)
    B = np.reshape(arr2, (-1, 28))
    name = labels[i]
    #print(B)
    imsave("E:/ML/Bangla_Digit/"+str(cnt_)+"-0"+str(name)+".jpeg", B)
    #print(arr2)
    """f= open("E:/ML/Bangla_Digit/"+str(cnt_)+"-"+str(name)+".txt","w+")
    cnt__=0"""
    """for j in arr2:
        cnt1=cnt1+1
    f.close()"""
    cnt_ = cnt_ + 1
    #print(arr2)
    print(name)

print(cnt_)


"""
Use Pro Football Reference for player stat
"""
import os
import datetime
from os import sys, path

import django
import requests

from bs4 import BeautifulSoup

sys.path.append(path.dirname(path.dirname(path.abspath(__file__))))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "fantasy_nfl.settings")
django.setup()

from general.models import *

FIELDS = ['team', 'pass_cmp', 'pass_att', 'pass_yds', 'pass_td', 'pass_int', 'pass_rating', 'pass_sacked',
          'pass_sacked_yds', 'pass_long', 'rush_att', 'rush_yds', 'rush_td', 'rush_long',
          'targets', 'rec', 'rec_yds', 'rec_td', 'rec_long', 'fumbles', 'fumbles_lost']

def sync(type_, val):
    # fit into roto
    val = val.strip().strip('@')
    conv = {
        'team': {
            'NOR': 'NO',
            'GNB': 'GB',
            'SFO': 'SF',
            'TAM': 'TB',
            'NWE': 'NE',
            'KAN': 'KC'
        },
        'name': {
        }
    }
    return conv[type_][val] if val in conv[type_] else val

def _C(val_dict, field):
    return float(val_dict.get(field, '0').strip() or '0')

def build_pos_dict(html):
    pos_dict = {}

    for table_id in ["home_starters", "vis_starters"]:
        tbl_text = html.split(f'
')[1].split('
')[0]
        soup = BeautifulSoup(tbl_text, "html.parser")
        table = soup.find("table", {"id": table_id})

        for player in table.find("tbody").find_all("tr"):
            uid = player.find("th", {"data-stat": "player"}).get('data-append-csv')
            pos = player.find("td", {"data-stat": "pos"}).text.strip()
            pos_dict[uid] = pos

            if player.get('class'): # ignore header
                break

    return pos_dict

def get_game_links(week):
    url = 'https://www.pro-football-reference.com/years/2019/week_{}.htm'.format(week)
    print("||" + url)
    r = requests.get(url).text
    soup = BeautifulSoup(r, "html.parser")

    links = []
    for ii in soup.find_all("td", {"class": "right gamelink"}):
        if ii.find('a').text.strip() == 'Final':
            link = ii.find('a').get('href')
            links.append(link)

    return links

def scrape(week):
    for game_link in get_game_links(week):
        url = 'https://www.pro-football-reference.com' + game_link
        print("|| - " + url)
        body = requests.get(url).text

        soup = BeautifulSoup(body, "html.parser")
        game_results = soup.find_all("div", {"class": "score"})
        home_score = int(game_results[0].text.strip())
        away_score = int(game_results[1].text.strip())

        # build the pos dict
        pos_dict = build_pos_dict(body)

        table = soup.find("table", {"id": "player_offense"})
        players = table.find("tbody").find_all("tr")

        home_team = sync('team', players[-1].find("td", {"data-stat": "team"}).text.strip())
        away_team = sync('team', players[0].find("td", {"data-stat": "team"}).text.strip())

        game_info = {
            home_team: [away_team, '', 'W' if home_score > away_score else 'T' if home_score == away_score else 'L'],
            away_team: [home_team, '@', 'L' if home_score > away_score else 'T' if home_score == away_score else 'W']
        }

        game_date = game_link[11:19]
        date = datetime.datetime.strptime(game_date, '%Y%m%d')

        # build data for players
        for player in players:
            if player.get('class'): # ignore header
                continue

            name = player.find("th", {"data-stat": "player"}).text.strip()
            name = sync('name', name)
            uid = player.find("th", {"data-stat": "player"}).get('data-append-csv')

            if not uid:
                continue

            defaults = {
                'name': name,
                'week_num': week
            }

            for ii in FIELDS:
                field = player.find("td", {"data-stat": ii}).text.replace('%', '').strip()
                if field:
                    defaults[ii] = field

            defaults['pos'] = pos_dict.get(uid, '')
            defaults['team'] = sync('team', defaults['team'])
            defaults['opp'] = game_info[defaults['team']][0]
            defaults['game_location'] = game_info[defaults['team']][1]
            defaults['game_result'] = game_info[defaults['team']][2]
            defaults['fpts'] = 0.1 * _C(defaults, 'rush_yds') + 6 * _C(defaults, 'rush_td') \
                             + 0.04 * _C(defaults, 'pass_yds') + 4 * _C(defaults, 'pass_td') \
                             - _C(defaults, 'pass_int') + 0.1 * _C(defaults, 'rec_yds') \
                             + 6 * _C(defaults, 'rec_td') + 0.5 * _C(defaults, 'rec')

            PlayerGame.objects.update_or_create(uid=uid, date=date, defaults=defaults)

if __name__ == "__main__":
    for week in range(1, 22):
        scrape(week)


from src.variant import var

import pytest

# 23
def z_one(a, c):
    a = a * 1.0
    try:
        result = (((1 + 6 * a * c) / (a ** 3 - 8 * c ** 3)) - (1 / (a - 2 * c))) / \
             (1 / (a ** 3 - 8 * c ** 3)) - (1 / (a ** 2 + 2 * a * c + 4 * c ** 2))
        print("No issues")
        return result
    except Exception:
        print("There was some issue in z_one function")
        raise

# 23
def z_two(a, c):
    try:
        result = 1 - 2 * c + a
        print("No issues")
        return result
    except Exception:
        print("There was some issue in z_two function")
        raise

def test_z_one():
    assert round(z_one(-3, 3), 3) == -80.037
    assert round(z_one(0, 4), 3) == -63.016
    with pytest.raises(Exception):
        assert z_one(-4, -2) == ArithmeticError
    assert round(z_one(0, -5), 3) == -99.010
    with pytest.raises(Exception):
        assert z_one(-4, -2) == ArithmeticError

def test_z_two():
    assert z_two(-3, 3) == -8
    assert z_two(0, 4) == -7
    assert z_two(-4, -2) == 1
    assert z_two(0, -5) == 11
    assert z_two(-4, -2) == 1

print("Variant #{}\n".format(var(82, 30)))
print("Test z_one:\n")
test_z_one()
print("\nTest z_two:\n")
test_z_two()


import logging

class Calculator:
    def __init__(self, amount: float = 0.0, interest: float = 0.0, down_payment: float = 0.0, term: float = 0.0):
        """Calculator(a, b, c, d) -> Calculator
        returns an instance of Calculator class"""
        self.logger = logging.getLogger(__name__)
        self.amount = amount
        self.interest = interest
        self.down_payment = down_payment
        self.term = term
        self.monthly_payment = 0.0
        self.total_interest = 0.0
        self.total_payment = 0.0

    def calculate(self):
        """ calculate() -> dict
        Return the dictionary containing self.monthly_payment, self.total_payment, self.total_interest
            if interest is equals to 0 it will raise a division by zero exception.
            return {
                'monthly payment': float,
                'total payment': float,
                'total interest': float
            }"""
        # P = Initial principal or loan amount
        p = int(self.amount - self.down_payment)
        # r = Interest rate per period
        r = self.interest / 12
        # n = Total number of payments or periods
        n = int(self.term * 12)

        self.monthly_payment = round(p * ((r * ((r + 1) ** n)) / (((r + 1) ** n) - 1)), 2)
        self.total_payment = round(self.monthly_payment * n, 2)
        self.total_interest = round(self.total_payment - p, 2)
        return {
            "monthly payment": self.monthly_payment,
            "total payment": self.total_payment,
            "total interest": self.total_interest
        }


# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import json
import getpass
import pathlib

import numpy as np
import pandas as pd

import heyhi.gsheets
import heyhi.run

import run

THIS_FILE = pathlib.Path(__file__).resolve()
THIS_DIR = THIS_FILE.parent

REDEFINES = [
    {},
    {"rollout.num_rollout_processes": 160},
    {"rollout.batch_size": 4},
    {"rollout.num_rollout_processes": 160, "optimizer.lr": 1e-4},
    {"rollout.num_rollout_processes": 160, "optimizer.lr": 1e-4, "critic_weight": 0.1},
    {
        "rollout.num_rollout_processes": 160,
        "optimizer.lr": 1e-4,
        "rollout.max_rollout_length": 60,
        "rollout.block_size": 7,
    },
    {
        "rollout.num_rollout_processes": 160,
        "optimizer.lr": 1e-4,
        "rollout.max_rollout_length": 60,
        "rollout.block_size": 7,
        "comment": "faster_rollout",
    },
    {
        "rollout.num_rollout_processes": 160,
        "rollout.max_rollout_length": 60,
        "rollout.block_size": 7,
        "comment": "faster_rollout",
    },
    {
        "rollout.num_rollout_processes": 160,
        "rollout.max_rollout_length": 100,
        "rollout.block_size": 5,
        "comment": "faster_rollout",
        "rollout.inference_batch_size": 32,
    },
]

def loguniform(rng, low, high):
    low, high = min(low, high), max(low, high)
    return np.exp(rng.uniform(np.log(low), np.log(high)))

def redefines_from_dict(redefines):
    chunks = []
    for k, v in redefines.items():
        chunks.append(f"{k}={v}")
    return chunks

def yield_sweep():
    cfg = THIS_DIR / "exploit.prototxt"
    for redefines in REDEFINES:
        redefines = dict(**redefines)
        redefines["I.launcher"] = "slurm_pascal"
        yield cfg, redefines

def get_logs_data(handle):
    metrics = {k: None for k in ["epoch", "reward/last", "loss/critic", "loss/entropy"]}
    result_path = handle.exp_path / "metrics.jsonl"
    if not result_path.exists():
        return metrics
    data = []
    with result_path.open() as stream:
        for line in stream:
            try:
                record = json.loads(line)
            except:
                break
            data.append(record)
    if not data:
        return metrics
    data = pd.DataFrame(data)

    metrics["epoch"] = data["epoch"].values.max()
    metrics["loss/critic"] = data["loss/critic"].dropna().values.min()
    if "loss/entropy" in data.columns:
        metrics["loss/entropy"] = data["loss/entropy"].dropna().values.min()
    if "reward/last" in data.columns:
        metrics["reward/last"] = data["reward/last"].dropna().values.max()
    return metrics

def main(mode, use_gsheet):
    handles = []
    sheet_file = "fairdip/%s" % THIS_DIR.name
    sheet_name = "%s" % (THIS_FILE.name.split(".")[0])
    exp_id = "%s_%s/rs" % (THIS_DIR.name, sheet_name)
    for cfg, override_dict in yield_sweep():
        h = heyhi.run.maybe_launch(
            run.main,
            exp_root=heyhi.run.get_exp_dir(heyhi.run.PROJECT_NAME),
            overrides=redefines_from_dict(override_dict),
            cfg=cfg,
            mode=mode,
            exp_id_pattern_override=f"%(prefix)s/{exp_id}_%(redefines)s_%(redefines_hash)s",
        )
        handles.append((h, override_dict))
    df = []
    col_order = {}
    for (h, od) in handles:
        datum = {}
        datum["status"] = str(h.get_status())
        datum.update(get_logs_data(h))
        datum.update(od)
        col_order.update(datum)
        # Not adding these feilds to col_order to out them at the end.
        datum["exp_id"] = h.exp_id.split("/")[-1].replace("sweep@launcher@slurm@", "")
        datum["job_id"] = h.maybe_get_job_id()
        datum["folder"] = h.exp_path
        df.append(datum)
    if df:
        # Puting meta fields to the col order.
        col_order.update(df[0])
        df = pd.DataFrame(df, columns=list(col_order))
        # df = df.sort_values("job_id")
        if use_gsheet:
            heyhi.gsheets.save_pandas_table(df, sheet_file, sheet_name, offset=1)
        del df["folder"]
        print(df.head(50).to_string(index=False))

if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument("--mode", choices=heyhi.MODES, default="gentle_start")
    parser.add_argument("--use_gsheet", type=int, default=int("yolo" == getpass.getuser()))
    main(**vars(parser.parse_args()))


import unittest
from datetime import datetime as dt
import dateutil.relativedelta
from royston.royston import Royston

test_doc = { 'id': '123', 'body': 'Random text string', 'date': dt.now() }
test_doc_2 = { 'id': '456', 'body': 'Antoher random string', 'date': dt.now() }
no_id_test_doc = { 'body': 'Another random string', 'date': dt.now() }
no_date_test_doc = { 'id': '123', 'body': 'Another random string' }

now = dt.now()

used_phrases = [
  ('antoher',),
  ('antoher', 'random'),
  ('antoher','random','string'),
  ('random',),
  ('random','string'),
  ('random','text'),
  ('random','text','string'),
  ('string',),
  ('text',),
  ('text','string')]

find_doc_options = {
  'start': now - dateutil.relativedelta.relativedelta(months = 1),
  'end': dt.now()
}

past_history_options = {
  'start': now - dateutil.relativedelta.relativedelta(years = 2), #moment().subtract(2, 'year'),
  'end': now - dateutil.relativedelta.relativedelta(years = 1) #moment().subtract(1, 'year')
}

class TestRoyston(unittest.TestCase):

    def test_normalise(self):
        r = Royston()
        # normalise: normalise a string
        self.assertEqual(r.normalise('My name is Ian'), ['name', 'ian'])

    def test_isupper(self):
        self.assertTrue('FOO'.isupper())
        self.assertFalse('Foo'.isupper())

    def test_split(self):
        s = 'hello world'
        self.assertEqual(s.split(), ['hello', 'world'])
        # check that s.split fails when the separator is not a string

    def test_constructor(self):
        r = Royston({ 'minTrendFreq': 5 })
        self.assertEqual(r.options['minTrendFreq'], 5)
        self.assertEqual(r.options['historyDays'], 90)

    def test_ingest_no_date(self):
        r = Royston({ 'minTrendFreq': 5 })
        with self.assertRaises(Exception) as context:
            r.ingest(no_date_test_doc)

    def test_ingest_no_id(self):
        r = Royston({ 'minTrendFreq': 5 })
        with self.assertRaises(Exception) as context:
            r.ingest(no_id_test_doc)

    def test_clean_date(self):
        r = Royston({})
        self.assertEqual(r.clean_date('2020-01-23 01:02:03'), dt(2020, 1, 23, 1, 2, 3))

    def test_ingest_the_same_doc_twice(self):
        r = Royston({ 'minTrendFreq': 5 })
        self.assertEqual(r.docs, {}) #maybe wrong syntax, as needs to be based on keys??

        r.ingest(test_doc)
        self.assertEqual(r.docs, { test_doc['id']: test_doc }) #maybe wrong syntax, as needs to be based on keys??

        with self.assertRaises(Exception) as context:
            r.ingest(test_doc)

    def test_ingest_all(self):
        """
        Test ingest_all ingests multiple documents
        """
        r = Royston({ 'minTrendFreq': 5 })
        self.assertEqual(r.docs, {})
        r.ingest_all([test_doc, test_doc_2])
        self.assertEqual(r.docs, { test_doc['id']: test_doc, test_doc_2['id']: test_doc_2 })

    def test_used_phrases(self):
        """
        Test used_phrases returns the correct phrases
        """
        r = Royston({ 'minTrendFreq': 5 })
        r.ingest_all([test_doc, test_doc_2])
        computed_phrases = r.used_phrases(find_doc_options['start'], find_doc_options['end'])
        # not perfect just looking at first element, but...
        computed_phrases = sorted(computed_phrases, key=lambda ngram: (ngram, len(ngram)))
        self.assertEqual(computed_phrases, used_phrases)
        self.assertEqual(computed_phrases, used_phrases)

        computed_phrases = r.used_phrases(find_doc_options['start'], find_doc_options['end'])

        # check the date filter is working to ignore stuff out of range
        r.used_phrases(find_doc_options['start'], find_doc_options['end'])
        self.assertEqual([], r.used_phrases(past_history_options['start'], past_history_options['end']))

if __name__ == '__main__':
    unittest.main()