# -*- coding: utf-8 -*-

from datetime import datetime

from odoo import models, fields, api,_
from datetime import datetime
from dateutil.relativedelta import relativedelta
from openerp.tools import DEFAULT_SERVER_DATE_FORMAT

class Asset(models.Model):
    _inherit = 'account.asset.asset'

    is_apply_notification = fields.Boolean('Apply Notification')
    emails = fields.Many2many('res.users', string="Emails")
    date_before_notification = fields.Integer("Date Before Notification", default=10)

    @api.multi
    def set_as_open(self):
        for rec in self.search([]):
            if rec.is_apply_notification:
                if rec.emails:
                    depreciation_date = datetime.now().date() + relativedelta(days=rec.date_before_notification)
                    domain = [
                        "|",
                        ('depreciation_date', '=', datetime.now().date().strftime(DEFAULT_SERVER_DATE_FORMAT)),
                        ('depreciation_date', '=', depreciation_date),
                        ('asset_id', '=', rec.id),
                    ]
                    base_url = self.env['ir.config_parameter'].sudo().get_param('web.base.url')
                    url = base_url + '/web#id=%s&view_type=form&model=account.asset.asset' % (rec.id)
                    for line in self.env['account.asset.depreciation.line'].search(domain):

                        email_from = self.company_id.email or 'Administrator <admin@example.com>'
                        for partner in rec.emails.mapped('partner_id'):
                            subject = 'Asset Notification'
                            message = """

                                                Dear %s,

                                               You have an Asset %s need to generate.

                                               Requestor: %s.

                                        """ % ( partner.name, url, line.asset_id.name, rec.responsible_id.user_id.partner_id.name)

                            mail_message = {
                                'subject': subject,
                                'email_from': email_from,
                                'body': message,
                                'partner_ids': [(6, 0, [partner.id])],
                                'needaction_partner_ids': [(6, 0, [partner.id])]
                            }
                            thread_pool = self.env['mail.message'].create(mail_message)
                            thread_pool.needaction_partner_ids = [(6, 0, [partner.id])]

                            email_out = {
                                'state': 'outgoing',
                                'subject': subject,
                                'body_html': '
%s
' % message,
                                'email_to': partner.email,
                                'email_from': email_from,
                            }
                            self.env['mail.mail'].create(email_out)


from setuptools import setup
import os

def read(fname):
    return open(os.path.join(os.path.dirname(__file__), fname)).read()

setup(
    name='pyurfa',
    version='0.0.1',
    description='Python api for UTM5 Billing',
    author='Ilia Gavrilin',
    author_email='gavrilin.ilia@gmail.com',
    url='http://bitbucket.org/chubi/pyurfa',
    keywords = "netup utm5 api",
    install_requires=[''],
    license='BSD',
    packages=['pyurfa'],
    long_description=read('README'),
    classifiers=[
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        ],
)


from flask import render_template, flash, redirect, url_for, request, jsonify, \
    current_app, g
from flask_babelex import lazy_gettext as _
from flask_login import current_user, login_required, login_user, logout_user
from rapidannotator import db
from rapidannotator import bcrypt
from rapidannotator.modules.clustering import blueprint
import datetime, os, base64, json, pandas
from rapidannotator.models import User, Experiment, Clustering, NotificationInfo
import requests as rq

@blueprint.route('/_setJob', methods=['GET', 'POST'])
def _setJob():
	experimentId = int(request.args.get('experimentId', None))
	userId = int(request.args.get('userId', None))

	clustering = Clustering.query.filter_by(experiment_id = experimentId).first()
	if clustering is None:
		clustering = Clustering(experiment_id = experimentId, user_id = userId, status=0)
		db.session.add(clustering)
		db.session.commit()

		response = {}
		response['success'] = True
		return jsonify(response)
	else:
		clustering = Clustering.query.filter_by(experiment_id = experimentId).first()
		response = {}
		response['success'] = False
		response['msg'] = "Clustering is under process ! Please Wait!"
		if clustering.status == 2:
			response['msg'] = "Clustering is already Done!"
		return jsonify(response)

@blueprint.route('/getJobData', methods=['GET'])
def index():
	clusters = Clustering.query.all()
	response = {}
	response['jobsData'] = []
	for cluster in clusters:
		if str(cluster.status) == '0':
			from rapidannotator import app
			experimentDIR = os.path.join(app.config['UPLOAD_FOLDER'], str(cluster.experiment_id))
			inputConcordance = os.path.join(experimentDIR, 'concordance.csv')
			data = pandas.read_csv(inputConcordance)
			first_pair = data['Screenshot'].tolist()
			second_pair = list(range(1, len(first_pair) + 1))
			dictionary = {'fileId': second_pair, 'imageURLS': first_pair, 'jobId': cluster.id, 'experiment_id': str(cluster.experiment_id)}
			response['jobsData'].append(dictionary)
	response['success'] = True
	return jsonify(response)

@blueprint.route('/setJobStatus', methods=['GET', 'POST'])
def setJobStatus():

	content = request.data
	content = content.decode('utf-8')
	content = eval(content)
	jobId = int(content['jobId'])

	clustering = Clustering.query.filter_by(id = jobId).first()
	if clustering is None:
		response = {}
		response['success'] = False
		return jsonify(response)

	if content['jobStatus'] == 'Processing':
		clustering.status = 1
		db.session.commit()

	response = {}
	response['success'] = True
	return jsonify(response)

@blueprint.route('/publishResults', methods=['GET', 'POST'])
def publishResults():

	content = request.data
	content = content.decode('utf-8')
	content = eval(content)

	from rapidannotator import app
	experimentDIR = os.path.join(app.config['UPLOAD_FOLDER'], str(content['experiment_id']))
	outJson = os.path.join(experimentDIR, 'output.json')

	out = content['largest1']
	out1 = sorted(range(len(out)), key=lambda k: out[k], reverse=True)
	content['sortOrder'] = out1

	with open(outJson, 'w') as json_file:
		json.dump(content, json_file, indent = 4, sort_keys=True)

	jobId = int(content['job_id'])
	clustering = Clustering.query.filter_by(id = jobId).first()

	if clustering is None:
		response = {}
		response['success'] = False
		return jsonify(response)

	clustering.status = 2
	db.session.commit()

	experimentId = int(content['experiment_id'])
	experiment_info = Experiment.query.filter_by(id=experimentId).first()
	experiment_owners = experiment_info.owners
	user = User.query.filter_by(id = int(clustering.user_id)).first()

	for owner in experiment_owners:
		owner.numNotif += 1
		message = 'The Clustering for the experiment ' + experiment_info.name + ' got completed by the ' + user.fullname
		notify = NotificationInfo()
		notify.user_id = owner.id
		notify.username = user.username
		notify.notification = message
		db.session.add(notify)
		db.session.commit()

	response = {}
	response['success'] = True
	return jsonify(response)

@blueprint.route('/getStatus', methods=['GET', 'POST'])
def getStatus():
	experiment_id = request.form['experiment_id']
	if experiment_id is not None:
		experiment_id = int(experiment_id)

	clustering = Clustering.query.filter_by(experiment_id = experiment_id, user_id = int(current_user.id)).first()
	if clustering is None:
		status = 0
	else:
		status = clustering.status

	response = {}
	response['success'] = True
	response['status'] = status
	return jsonify(response)

@blueprint.route('/toggleDisplay', methods=['GET', 'POST'])
def toggleDisplay():
	experiment_id = request.args.get('experiment_id', None)
	option = request.args.get('option')
	clustering = Clustering.query.filter_by(experiment_id = experiment_id, user_id = int(current_user.id)).first()
	response = {}
	if clustering is None:
		response['success'] = False
	else:
		if option == "Yes":
			clustering.display = 1
		else:
			clustering.display = 0
		db.session.commit()
		response['success'] = True
	return jsonify(response)


"""Core recipes for EMT"""
from __future__ import annotations

import covalent as ct
from ase import Atoms
from ase.calculators.lj import LennardJones

from quacc.schemas.ase import summarize_opt_run, summarize_run
from quacc.util.atoms import copy_atoms
from quacc.util.calc import run_ase_opt, run_calc
from quacc.util.dicts import merge_dicts

# NOTE: This set of minimal recipes is mainly for demonstration purposes

@ct.electron
def static_job(
    atoms: Atoms,
    lj_kwargs: dict | None = None,
) -> dict:
    """
    Function to carry out a static calculation.

    Parameters
    ----------
    atoms
        .Atoms object
    lj_kwargs
        Dictionary of custom kwargs for the LJ calculator.

    Returns
    -------
    summary
        Summary of the run.
    """

    lj_kwargs = lj_kwargs or {}
    input_atoms = copy_atoms(atoms)

    defaults = {"epsilon": 1.0, "sigma": 1.0}
    flags = merge_dicts(defaults, lj_kwargs)

    atoms.calc = LennardJones(**flags)
    atoms = run_calc(atoms)

    return summarize_run(
        atoms, input_atoms=input_atoms, additional_fields={"name": "LJ Static"}
    )

@ct.electron
def relax_job(
    atoms: Atoms,
    fmax: float = 0.01,
    max_steps: int = 1000,
    optimizer: str = "FIRE",
    lj_kwargs: dict | None = None,
    opt_kwargs: dict | None = None,
) -> dict:
    """
    Function to carry out a geometry optimization.

    Parameters
    ----------
    atoms
        .Atoms object
    fmax
        Tolerance for the force convergence (in eV/A).
    max_steps
        Maximum number of steps to take.
    optimizer
        .Optimizer class to use for the relaxation.
    lj_kwargs
        Dictionary of custom kwargs for the LJ calculator.
    opt_kwargs
        Dictionary of kwargs for the optimizer.

    Returns
    -------
    summary
        Summary of the run.
    """

    lj_kwargs = lj_kwargs or {}
    opt_kwargs = opt_kwargs or {}

    defaults = {"epsilon": 1.0, "sigma": 1.0}
    flags = merge_dicts(defaults, lj_kwargs)

    atoms.calc = LennardJones(**flags)
    traj = run_ase_opt(
        atoms,
        fmax=fmax,
        max_steps=max_steps,
        optimizer=optimizer,
        opt_kwargs=opt_kwargs,
    )

    return summarize_opt_run(
        traj, atoms.calc.parameters, additional_fields={"name": "LJ Relax"}
    )


"""This processor parses currency fields and returns floats."""
from logging import warning

from datapackage_pipelines.wrapper import ingest, spew
from common.utilities import process, format_to_json

def parse_currencies(row, fields=None, characters=None):
    """Clean up and convert currency fields to floats."""

    assert fields, 'Missing `fields` parameter'
    assert characters, 'Missing `characters` parameter'

    for key in fields:
        if row[key] is not None:
            row[key] = str(row[key])

            if not row[key].strip():
                row[key] = None
            else:
                try:
                    row[key] = float(row[key]
                                     .replace(characters['currency'], '')
                                     .replace(characters['grouping'], '')
                                     .replace(characters['decimal'], '.')
                                     .strip())
                except ValueError as error:
                    warning('%s in row\n%s', error, format_to_json(row))
    return row

if __name__ == '__main__':
    parameters, datapackage_, resources = ingest()
    new_resources_ = process(resources, parse_currencies, **parameters)
    spew(datapackage_, new_resources_)


import h5py, os
import matplotlib.pyplot as plt
from tensorflow import keras
import tensorflow as tf
import numpy as np
from sequence import DataSequenceFromFile
from metrics import fig_to_np

model = keras.models.load_model('../models/models/model_0524_144435')

def plot_value_array(prediction, true_label):
    plt.grid(False)
    plt.xticks(range(len(true_label)))
    plt.yticks([])
    plt.ylim([0, 1])
    plt.bar(range(len(true_label)), true_label, color='#44884444')
    plt.bar(range(len(true_label)), prediction, color='#444444ff')

with h5py.File('../data/preprocessed/data_test_432.hdf5', 'r') as f:
    for sample in range(10):
        print('Prediction of sample #' + str(sample))
        img = (np.expand_dims(f['X'][sample], 0))
        predictions_single = model.predict(img)[0]
        plot_value_array(predictions_single, f['y'][sample])
        _ = plt.xticks(range(14))
        plt.show()

    model.evaluate(f['X'][...], f['y'][...])


# -*- coding: utf-8 -*-
import scrapy

from tutorial.items import TutorialItem
class DmonSpiderSpider(scrapy.Spider):
    name = 'dmon_spider'
    allowed_domains = ['dmoz.org']
    start_urls = ['https://www.amazon.cn/s?ie=UTF8&search-type=ss&index=books&field-keywords=Python%E6%95%B0%E6%8D%AE%E5%88%86%E6%9E%90%E4%B8%8E%E6%8C%96%E6%8E%98%E5%AE%9E%E6%88%98&tag=baiduiclickcn-23&ref=Book_17417_zhj_5279']

    def parse(self, response):
        # filename = response.url.rsplit('/')[2] + '.html'
        # with open(filename,'wb') as e:
        #     e.write(response.body)
        lis =response.xpath('//*[@id="result_1"]/div/div/div/div[2]/div[1]/div[1]/a/h2')
        for li in lis:
            item = TutorialItem()
            item['title'] = li.xpath('a/text()').extract()
            item['link'] = li.xpath('a/@href')
            item['desc'] = li.xpath('text()')
            yield item


import torch
import os
import argparse

from helpers import *
from model import *

"""
This class is for generating lyrics from a trained model.
"""

class Generator:

    def __init__(self, filename):
        self.decoder = torch.load(filename)

    def generate(self, prime_str='A', predict_len=100, temperature=0.8, cuda=False):
        hidden = self.decoder.init_hidden(1)
        prime_input = Variable(char_tensor(prime_str).unsqueeze(0))

        if cuda:
            hidden = hidden.cuda()
            prime_input = prime_input.cuda()
        predicted = prime_str

        # Use priming string to "build up" hidden state
        for p in range(len(prime_str) - 1):
            _, hidden = self.decoder(prime_input[:,p], hidden)

        inp = prime_input[:,-1]

        for p in range(predict_len):
            output, hidden = self.decoder(inp, hidden)

            # Sample from the network as a multinomial distribution
            output_dist = output.data.view(-1).div(temperature).exp()
            top_i = torch.multinomial(output_dist, 1)[0]

            # Add predicted character to string and use as next input
            predicted_char = all_characters[top_i]
            predicted += predicted_char
            inp = Variable(char_tensor(predicted_char).unsqueeze(0))
            if cuda:
                inp = inp.cuda()

        print('>>> generated lyrics:')
        print(predicted)


"""columns weight, height, length, width altered for not null

Revision ID: 00fbdb3c3ccc
Revises: 9a6c8eb30815
Create Date: 2021-10-18 14:33:11.779427

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '00fbdb3c3ccc'
down_revision = '9a6c8eb30815'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.execute("UPDATE products SET weight = '' WHERE weight is NULL")
    op.execute("UPDATE products SET height = '' WHERE height is NULL")
    op.execute("UPDATE products SET length = '' WHERE length is NULL")
    op.execute("UPDATE products SET width = '' WHERE width is NULL")
    op.alter_column('products', 'weight',
               existing_type=sa.VARCHAR(length=16),
               nullable=False)
    op.alter_column('products', 'height',
               existing_type=sa.VARCHAR(length=16),
               nullable=False)
    op.alter_column('products', 'length',
               existing_type=sa.VARCHAR(length=16),
               nullable=False)
    op.alter_column('products', 'width',
               existing_type=sa.VARCHAR(length=16),
               nullable=False)
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.alter_column('shipping_companies', 'minimum_shipping_price',
               existing_type=sa.INTEGER(),
               nullable=False)
    op.alter_column('products', 'width',
               existing_type=sa.VARCHAR(length=16),
               nullable=True)
    op.alter_column('products', 'length',
               existing_type=sa.VARCHAR(length=16),
               nullable=True)
    op.alter_column('products', 'height',
               existing_type=sa.VARCHAR(length=16),
               nullable=True)
    op.alter_column('products', 'weight',
               existing_type=sa.VARCHAR(length=16),
               nullable=True)
    # ### end Alembic commands ###


import sys
import OpenGL.GL as gl
import OpenGL.GLUT as glut

def display():
	glut.glutSwapBuffers()

def reshape(width, height):
	gl.glViewport(0, 0, width, height)

def keyboard(key, x, y):
	if key == b'\x1b':
		sys.exit(0)

glut.glutInit()
glut.glutInitDisplayMode(glut.GLUT_DOUBLE | glut.GLUT_RGBA)
glut.glutCreateWindow("glut window")
glut.glutReshapeWindow(512, 512)
glut.glutReshapeFunc(reshape)
glut.glutDisplayFunc(display)
glut.glutKeyboardFunc(keyboard)
glut.glutMainLoop()

# need to request program and shader slots from the GPU
program = gl.glCreateProgram()
vertex = gl.glCreateShader(gl.GL_VERTEX_SHADER)
fragment = gl.glCreateShader(gl.GL_FRAGMENT_SHADER)

# ask for the compilation of our shaders into GPU objects and we log for any error
vertex_code = """
	attribute vec2 position;
	void main() { gl_Position = vec4(position, 0.0, 1.0); } """

fragment_code = """
	void main() { gl_FragColor = vec4(1.0, 0.0, 0.0, 1.0); } """

# Set shaders source
gl.glShaderSource(vertex, vertex_code)
gl.glShaderSource(fragment, fragment_code)

# Compile shaders
gl.glCompileShader(vertex)
if not gl.glGetShaderiv(vertex, gl.GL_COMPILE_STATUS):
    error = gl.glGetShaderInfoLog(vertex).decode()
    print(error)
    raise RuntimeError("Vertex shader compilation error")

gl.glCompileShader(fragment)
if not gl.glGetShaderiv(fragment, gl.GL_COMPILE_STATUS):
    error = gl.glGetShaderInfoLog(fragment).decode()
    print(error)
    raise RuntimeError("Fragment shader compilation error")

# link our two objects in into a program and again
# we check for errors during the process
gl.glAttachShader(program, vertex)
gl.glAttachShader(program, fragment)
gl.glLinkProgram(program)

if not gl.glGetProgramiv(program, gl.GL_LINK_STATUS):
    print(gl.glGetProgramInfoLog(program))
    raise RuntimeError('Linking error')


import csv

from rx import Observable

filename = '100 Sales Records.csv'

def read_csv_file(filename):
    with open(filename) as file:
        lines = file.readlines()
        return lines[0], csv.reader(lines[1:])

if __name__ == '__main_1_':
    Observable.from_([{'1':2}, {'2':7}, {'5':3}]).merge_all().subscribe(lambda g: print(g))

if __name__ == '__main__':
    header, reader = read_csv_file(filename)
    header = header.split(',')
    # f = list(reader)[0]
    #
    # for i, t in zip(f, header):
    #     print(i, t)

    # print({k:v for k,v in zip(header, f)})

    def to_d(row):
        print(row)
        print(len(header), len(row))
        d = {}
        for k,v in zip(header, row):
            # print(k, ' == ', v)
            d[k] = v

        print(d)
        return d

    def ff(row):
        print('row', row)
        return row.to_list()
        # Observable.from_(row).merge().subscribe(lambda g: print('g', g))
        Observable.merge(row[:]).subscribe(lambda g: print('g', g))
        # return row.flat_map(lambda i: i.flat_map(lambda y:y.flat_map(lambda o:o)))
        return Observable.merge(row[:])
        return row.merge_all()

    def subdictes_to_dict(row):
        d = {}

        for x in row:
            d.update(x)

        return d

    source = Observable.from_iterable(reader)\
        .map(lambda row: Observable.zip(Observable.from_(header), row, lambda k,v: {k:v}))\
        .map(lambda x: x.to_list().map(subdictes_to_dict))#.subscribe(lambda r: print(r, end="\n\n\n"))

    print(source.flat_map(lambda x: x).subscribe(lambda x: print(x)))
    # Observable.merge(source[:]).merge_all()


import numpy as np
import math
import matplotlib.pyplot as plt

def displayData (X):
    example_width = int(round(math.sqrt(np.size(X,1))))
    plt.gray()
    m,n = X.shape
    example_height = int(n / example_width)

    display_rows = int(math.floor(np.sqrt(m)))
    display_cols = int(math.ceil(m / display_rows))
    pad = 1;

    display_array = np.ones((pad + display_rows * (example_height + pad), \
                             pad + display_cols * (example_width + pad)))
    curr_ex = 0
    for j in range(0, display_rows):
        for i in range (0, display_cols):
            if curr_ex >= m:
                break
            max_val = max(abs(X[curr_ex, :]))
            rows = pad + j * (example_height + pad) + np.array(range(example_height))
            cols = pad + i * (example_width + pad) + np.array(range(example_width))
            display_array[rows[0]:rows[-1]+1 , cols[0]:cols[-1]+1] = \
            np.reshape(X[curr_ex-1, :], (example_height, example_width), order="F") \
            / max_val
            curr_ex = curr_ex + 1
        if curr_ex >= m:
            break
    h = plt.imshow(display_array, vmin=-1, vmax=1)
    # Do not show axis
    plt.axis('off')
    return plt, h, display_array


#!/usr/bin/env python3

import torch

from .. import settings
from ..utils.cholesky import psd_safe_cholesky
from ..utils.memoize import cached
from .block_lazy_tensor import BlockLazyTensor
from .non_lazy_tensor import NonLazyTensor
from .root_lazy_tensor import RootLazyTensor

class BlockDiagLazyTensor(BlockLazyTensor):
    """
    Represents a lazy tensor that is the block diagonal of square matrices.
    The :attr:`block_dim` attribute specifies which dimension of the base LazyTensor
    specifies the blocks.
    For example, (with `block_dim=-3` a `k x n x n` tensor represents `k` `n x n` blocks (a `kn x kn` matrix).
    A `b x k x n x n` tensor represents `k` `b x n x n` blocks (a `b x kn x kn` batch matrix).

    Args:
        :attr:`base_lazy_tensor` (LazyTensor or Tensor):
            Must be at least 3 dimensional.
        :attr:`block_dim` (int):
            The dimension that specifies the blocks.
    """
    @property
    def num_blocks(self):
        return self.base_lazy_tensor.size(-3)

    def _add_batch_dim(self, other):
        *batch_shape, num_rows, num_cols = other.shape
        batch_shape = list(batch_shape)

        batch_shape.append(self.num_blocks)
        other = other.view(*batch_shape, num_rows // self.num_blocks, num_cols)
        return other

    def _get_indices(self, row_index, col_index, *batch_indices):
        # Figure out what block the row/column indices belong to
        row_index_block = row_index.div(self.base_lazy_tensor.size(-2))
        col_index_block = col_index.div(self.base_lazy_tensor.size(-1))

        # Find the row/col index within each block
        row_index = row_index.fmod(self.base_lazy_tensor.size(-2))
        col_index = col_index.fmod(self.base_lazy_tensor.size(-1))

        # If the row/column blocks do not agree, then we have off diagonal elements
        # These elements should be zeroed out
        res = self.base_lazy_tensor._get_indices(row_index, col_index, *batch_indices, row_index_block)
        res = res * torch.eq(row_index_block, col_index_block).type_as(res)
        return res

    def _remove_batch_dim(self, other):
        shape = list(other.shape)
        del shape[-3]
        shape[-2] *= self.num_blocks
        other = other.contiguous().view(*shape)
        return other

    def _size(self):
        shape = list(self.base_lazy_tensor.shape)
        shape[-2] *= shape[-3]
        shape[-1] *= shape[-3]
        del shape[-3]
        return torch.Size(shape)

    def diag(self):
        res = self.base_lazy_tensor.diag().contiguous()
        return res.view(*self.batch_shape, self.size(-1))

    def inv_quad_logdet(self, inv_quad_rhs=None, logdet=False, reduce_inv_quad=True):
        if inv_quad_rhs is not None:
            inv_quad_rhs = self._add_batch_dim(inv_quad_rhs)
        inv_quad_res, logdet_res = self.base_lazy_tensor.inv_quad_logdet(
            inv_quad_rhs, logdet, reduce_inv_quad=reduce_inv_quad
        )
        if inv_quad_res is not None and inv_quad_res.numel():
            if reduce_inv_quad:
                inv_quad_res = inv_quad_res.view(*self.base_lazy_tensor.batch_shape)
                inv_quad_res = inv_quad_res.sum(-1)
            else:
                inv_quad_res = inv_quad_res.view(*self.base_lazy_tensor.batch_shape, inv_quad_res.size(-1))
                inv_quad_res = inv_quad_res.sum(-2)
        if logdet_res is not None and logdet_res.numel():
            logdet_res = logdet_res.view(*logdet_res.shape).sum(-1)
        return inv_quad_res, logdet_res

    @cached(name="root_decomposition")
    def root_decomposition(self):
        if settings.fast_computations.covar_root_decomposition.on():
            res = self.__class__(self.base_lazy_tensor.root_decomposition().root)
        else:
            chol = psd_safe_cholesky(self.base_lazy_tensor.evaluate())
            res = self.__class__(NonLazyTensor(chol))
        return RootLazyTensor(res)


# MIT License
#
# Copyright (c) 2019 Creative Commons
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

"""from __future__ imports must occur at the beginning of the file. DO NOT CHANGE!"""
from __future__ import annotations

import time
from typing import List

from selenium import webdriver
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait

from selenium.common.exceptions import TimeoutException
from selenium.common.exceptions import NoSuchElementException

from . import Person_Info

from . import Path_To_Element_By
from ..DOM.javascript import JS

class Person(object):
    """Class Person to target the person element on the page."""

    def __init__(self: Person, driver: webdriver.Chrome) -> None:
        """Constructor method to initialize the driver instance and element count.

        :Args:
            - self: {Person} self.
            - driver: {webdriver.Chrome} chromedriver instance.

        :Raises:
            - {Exception} if the driver object in not identified.
        """
        if not isinstance(driver, webdriver.Chrome):
            raise Exception("'%(driver_type)s' object is not a 'webdriver' object" % {
                            "driver_type": type(driver)})

        self._driver = driver
        self.__suggestion_box_element_count = 0

    def __load_page(self: Person) -> None:
        """Private method __load_page() loads the page by scrolling it down.

        :Args:
            - self: {Person} self

        :Returns:
            - {None}
        """
        _js = JS(self._driver)

        _old_page_offset = _js.get_page_y_offset()
        _new_page_offset = _js.get_page_y_offset()

        while _old_page_offset == _new_page_offset:
            _js.scroll_bottom()
            time.sleep(1)
            _new_page_offset = _js.get_page_y_offset()

    def get_suggestion_box_element(self: Person):
        """Method get_suggestion_box_element() an object containing the person details
        including the connect button to connect with the person.

        :Args:
            - self: {Person} self.

        :Returns:
            - {Person_Info} person object.
        """
        def get_suggestion_box_person_li() -> webdriver.Chrome:
            """Nested function get_suggestion_box_person() finds the actual element that
            wraps the person on the page.

            :Args:
                - {None}

            :Return:
                - {webdriver.Chrome} element that wraps the person.
            """
            # Using parent function 'self' variable
            nonlocal self

            # Traget the element using its root xpath
            _xpath: str = Path_To_Element_By.SUGGESTION_BOX_ELEMENT_XPATH
            # Update the xpath every time the function is called to target the next element
            _xpath = _xpath[:-3] + '[' + \
                str(self.__suggestion_box_element_count + 1) + ']'

            if self.__suggestion_box_element_count == 0:
                self.__load_page()

            self.__suggestion_box_element_count += 1

            while True:
                try:
                    return WebDriverWait(self._driver, 60).until(
                        EC.presence_of_element_located((By.XPATH, _xpath))
                    )
                except (TimeoutException, NoSuchElementException) as error:
                    if isinstance(error, TimeoutException):
                        self.__load_page()
                    continue

        def transform_to_object(li: webdriver.Chrome) -> Person_Info:
            """Nested function transform_to_object() gets the person's details from the element
            that wraps the person and transform into an Person_Info object.

            :Args:
                - li: {webdriver.Chrome} element to get the person details from

            :Returns:
                - {Person_Info}
            """
            # Target the info container that is inside of the element we are given
            _info_container: webdriver.Chrome = li.find_element_by_css_selector(
                "div[class='discover-entity-type-card__info-container']")
            # Target the anchor tag from the info container to later get the image url
            # and the profile url of the person
            _anchor_tag: webdriver.Chrome = _info_container.find_element_by_tag_name(
                "a")
            # Target the image element to get the image url later
            _img_tag: webdriver.Chrome = _anchor_tag.find_element_by_tag_name(
                "img")
            # Target the bottom container inside of the info container to target the connect
            # button
            _bottom_container: webdriver.Chrome = li.find_element_by_css_selector(
                "div[class^='discover-entity-type-card__bottom-container']")
            # Target the footer element inside of the bottom container to get the connect button
            _footer: webdriver.Chrome = _bottom_container.find_element_by_tag_name(
                "footer")

            # Get the person's name from the element inside of the anchor tag
            _person_name: str = _anchor_tag.find_element_by_css_selector(
                "span[class^='discover-person-card__name']").text
            # Get the person's occupation from the element inside of the anchor tag
            _person_occupation: str = _anchor_tag.find_element_by_css_selector(
                "span[class^='discover-person-card__occupation']").text
            # Get the person's photo url from the image tag
            _person_photo_url: str = _img_tag.get_attribute("src")
            # Get the person's profile url from the anchor tag
            _person_profile_url: str = "%(profile_path)s" % {
                "profile_path": _anchor_tag.get_attribute("href")}
            # Get the connect button from the footer element
            _person_connect_button: str = _footer.find_element_by_css_selector(
                f"button[aria-label^='Invite']")

            return Person_Info(
                name=_person_name,
                occupation=_person_occupation,
                profile_url=_person_profile_url,
                photo_url=_person_photo_url,
                connect_button=_person_connect_button
            )

        return transform_to_object(get_suggestion_box_person_li())

    def get_search_results_elements(self: Person) -> List[Person_Info]:
        def get_search_results_person_lis() -> List[webdriver.Chrome]:
            nonlocal self

            _target: int = 1

            # Traget the element using its root xpath
            _xpath: str = Path_To_Element_By.SEARCH_RESULTS_PEOPLE_XPATH

            _search_results_person_lis: List[webdriver.Chrome] = []

            while True:
                # Update the xpath every time the function is called to target the next element
                _xpath = _xpath[:-3] + '[' + str(_target) + ']'
                try:
                    _search_results_person_lis.append(WebDriverWait(self._driver, 60).until(
                        EC.presence_of_element_located((By.XPATH, _xpath))
                    ))
                except (TimeoutException, NoSuchElementException) as error:
                    if isinstance(error, NoSuchElementException):
                        break
                    continue
                _target += 1

            return _search_results_person_lis

        def transform_to_object(lis: List[webdriver.Chrome]) -> List[Person_Info]:
            _person_infos: List[Person_Info] = []

            for li in lis:
                _entity_result_item_container: webdriver.Chrome = li.find_element_by_css_selector(
                    "div[class='entity_result']").find_element_by_css_selector(
                        "div[class='entity-result__item']")
                _entity_result_image_container: webdriver.Chrome = _entity_result_item_container.find_element_by_css_selector(
                    "div[class='entity-result__image']")

                _entity_result_anchor_tag: webdriver.Chrome = _entity_result_image_container.find_element_by_tag_name(
                    "a")
                _person_profile_url: str = _entity_result_anchor_tag.get_attribute(
                    "href")
                _entity_result_img_tag: webdriver.Chrome = _entity_result_image_container.find_element_by_tag_name(
                    "img")
                _person_photo_url: str = _entity_result_img_tag.get_attribute(
                    "src")

                _entity_result_content_container: webdriver.Chrome = _entity_result_item_container.find_element_by_css_selector(
                    "div[class^='entity-result__content']")
                _person_occupation: str = _entity_result_content_container.find_element_by_css_selector(
                    "div[class^='entity-result__primary-subtitle']")
                _person_location: str = _entity_result_content_container.find_element_by_css_selector(
                    "div[class^='entity-result__secondary-subtitle']")
                _person_summary: str = _entity_result_content_container.find_element_by_css_selector(
                    "p[class^='entity-result__summary']")

                _entity_result_content_anchor_tag: webdriver.Chrome = _entity_result_content_container.find_element_by_tag_name(
                    "a")
                _person_name: str = _entity_result_content_anchor_tag.text

                _person_connect_button: webdriver.Chrome = _entity_result_item_container.find_element_by_css_selector(
                    "div[class^='entity-result__actions']").find_element_by_tag_name("button")

                _person_infos.append(Person_Info(
                    name=_person_name,
                    occupation=_person_occupation,
                    photo_url=_person_photo_url,
                    profile_url=_person_profile_url,
                    location=_person_location,
                    summary=_person_summary,
                    connect_button=_person_connect_button
                ))

            return _person_infos

        return transform_to_object(get_search_results_person_lis())


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Script that show abundances Vs time in log-log for a given list of species

import os, pdb
import numpy as np
import sys # to be able to retrieve arguments of the script
import pylab as pl
from matplotlib.ticker import FormatStrFormatter, ScalarFormatter

###############################################
## Beginning of the program
###############################################

OUTPUT_EXTENSION = 'pdf' # default value in bitmap, because vectoriel can take time and space if there is a lot of data

# By default, 1, but represent the column containing abundances in each file.
## If one, this is either the first 1D point, or the only one, if this is not a 1D simulation

species_name = None

isProblem = False
problem_message = """AIM : For a given species, display the most important reaction
both for production and destruction. Display the evolution of theses importances
with time.

The script can take various arguments :
(no spaces between the key and the values, only separated by '=')
 * species=CO : the species name you want to plot the most important reactions
 * ext=%s : The extension for the output files
 * help : display a little help message on HOW to use various options.

EXAMPLE:
 > nautilus-trace-species.py species=CO""" % (OUTPUT_EXTENSION)

value_message = "/!\ Warning: %s does not need any value, but you defined '%s=%s' ; value ignored."

# We get arguments from the script
for arg in sys.argv[1:]:
  try:
    (key, value) = arg.split("=")
  except:
    key = arg
    value = None
  if (key == 'ext'):
    OUTPUT_EXTENSION = value
  elif (key == 'species'):
    species_name = value
  elif (key == 'help'):
    isProblem = True
    if (value != None):
      print(value_message % (key, key, value))
  else:
    print("the key '%s' does not match" % key)
    isProblem = True

if (species_name == None):
  isProblem = True

if isProblem:
  print(problem_message)
  exit()

####################
# We declare arrays to store read informations
####################

filename = 'trace_dest_%s.percentage' % species_name
if not(os.path.isfile(filename)):
  print("Error: The file %s doesn't exists." % filename)
  print("Please run nautilus_trace_major for the given reaction beforehand.")
  exit()

time = []
production_fraction = []
destruction_fraction = []

# Read production reactions
object_file = open('trace_prod_%s.percentage' % species_name, 'r')

# First header
dummy = object_file.readline()

# reaction IDs
line = object_file.readline()
reaction_prod_IDs = [word for word in line.split()]

nb_prod_reactions = len(reaction_prod_IDs)
for i in range(nb_prod_reactions):
  production_fraction.append([])
# 2nd header
dummy = object_file.readline()

# Actual datas
lines = object_file.readlines()

for line in lines:
  words = line.split()

  time.append(float(words[0]))
  for i in range(nb_prod_reactions):
    production_fraction[i].append(float(words[i+1]))

object_file.close()

# Read destruction reactions

time = [] # We reset the time array because we assume the same times are valid both for production AND destruction !
object_file = open('trace_dest_%s.percentage' % species_name, 'r')

# First header
dummy = object_file.readline()

# reaction IDs
line = object_file.readline()
reaction_dest_IDs = [word for word in line.split()]

nb_dest_reactions = len(reaction_dest_IDs)
for i in range(nb_dest_reactions):
  destruction_fraction.append([])
# 2nd header
dummy = object_file.readline()

# Actual datas
lines = object_file.readlines()

for line in lines:
  words = line.split()

  time.append(float(words[0]))
  for i in range(nb_dest_reactions):
    destruction_fraction[i].append(float(words[i+1]))

object_file.close()

#~ pdb.set_trace()

fig = pl.figure()
pl.clf()
fig.suptitle("Main production and destruction reactions for %s" % species_name)

#~ fig.subplots_adjust(left=0.12, bottom=0.1, right=0.96, top=0.95, wspace=0.26, hspace=0.26)
# We create subplots. add_subplot(2, 3, 1) means we have 2 lines, 3 columns,
# and that the active plot is the first, starting from top left (for 6 plots in total)

# Plot production
plot_prod = fig.add_subplot(2, 1, 1)

plot_prod.set_xscale("log")
plot = plot_prod.plot

for reaction in range(nb_prod_reactions):
  plot(time, production_fraction[reaction], label="ID=%s" % reaction_prod_IDs[reaction])

plot_prod.set_xlabel("Time [years]")
plot_prod.set_ylabel("Production fraction [%]")
plot_prod.grid(True)

#~ timeFormat = FormatStrFormatter("%.3g")
#~ plot_prod.xaxis.set_major_formatter(timeFormat)

plot_prod.legend()

# Plot destruction
plot_dest = fig.add_subplot(2, 1, 2)

plot_dest.set_xscale("log")
plot = plot_dest.plot

for reaction in range(nb_dest_reactions):
  plot(time, destruction_fraction[reaction], label="ID=%s" % reaction_dest_IDs[reaction])

plot_dest.set_xlabel("Time [years]")
plot_dest.set_ylabel("Destruction fraction [%]")
plot_dest.grid(True)

plot_dest.legend()

#~ percentage_format = FormatStrFormatter("%3.1f")
percentage_format = ScalarFormatter(useOffset=False)

plot_prod.yaxis.set_major_formatter(percentage_format)
plot_dest.yaxis.set_major_formatter(percentage_format)

nom_fichier_plot = "major_reaction_%s" % species_name
fig.savefig('%s.%s' % (nom_fichier_plot, OUTPUT_EXTENSION), format=OUTPUT_EXTENSION)

#~ pdb.set_trace()

pl.show()


# Import the necessary modules
import pandas as pd  # for handling data
import requests # for pulling pages from the internet
from bs4 import BeautifulSoup  # for scraping and processing pages
import numpy as np # for statistical calculations
import matplotlib.pyplot as plt # for plotting histograms
from matplotlib.pyplot import figure # For plotting feature importance
from pandas.plotting import scatter_matrix # for scatter matrices

import sklearn
from sklearn.preprocessing import StandardScaler # For performing z-normalizations
from sklearn.model_selection import train_test_split # For splitting data
from sklearn.tree import DecisionTreeClassifier # Decision tree classifier
from sklearn.ensemble import RandomForestClassifier # Random forest classifier
from sklearn import svm # for Support Vector Machine classifier
from sklearn.metrics import accuracy_score # for generating accuracy scores

from sklearn.metrics import * # for model evaluation

#################  EXTRACT DATA AND PROCESS COLUMN HEADERS ####################

# Read the data from the website into a Pandas DataFrame.
df = pd.read_csv("https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data")

# Pull the page with the column names and metadata.
cols_url = requests.get("https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.names")

# Use BS4 to split the page into a list of strings
soup = BeautifulSoup(cols_url.content, "lxml").text.split('\n')

# Inspecting the HTML of the page reveals the headers are in dict format on lines 96-110
# Pull the lines with column strings from the HTML into a list.
col_strings = soup[96:110]

# Create function to the lines and rebuild them into a dictionary with column names and possible values.
def strings_to_dicts(s_list):
    col_dict = {}
    for item in s_list:
        entry = item.split(':')
        item = entry[0]
        key = entry[1]
        col_dict[item] = key
    return col_dict

# Run html stripping function.
colsd = strings_to_dicts(col_strings)

# Extract column names and convert to list format.
cols = list(colsd.keys())

# Put together missing column names.
cols = cols + ['income_cat']

# Provide the column names for the DataFrame using the column list.
df.columns = cols

############  CREATE INFO FUNCTION AND PERFORM DATA EXPLORATION #########

# Define function to get information from the DataFrame.
def df_info(df, dv = None):
    """Automatically combine dataset information into one dataframe.
       Use both pandas and numpy methods of getting
       max and min to help catch bad cell values."""

    # Create empty DataFrame for info to go in.
    info= pd.DataFrame()

    # Insert the datatypes.
    info['dtype'] = df.dtypes

    # Compute number of rows with one or more fields mising
    missing_rows_ct = df.shape[0] - df.dropna().shape[0]

    # Count non-null values in each column
    info['#values'] = df.count()

    # Count the null values and and insert them.
    info['is_null'] = df.isnull().sum()

    # Caclulate the percentage of the field is missing.
    info['pct_null'] = info['is_null'] / len(df)

    # Proportion of total rows missing
    info['pct_null_rows'] = info['is_null'] / missing_rows_ct

    # Count the duplicates and insert them.
    dupcount = df.duplicated().sum()

    # Run describe function and then transpose the values.
    stats = df.describe().transpose()

    # Transpose the stats DataFrame and then join it with the info DataFrame.
    info = info.join(stats,how='outer',rsuffix='_np')

    # Create blank dictionary for categorical column information.
    cat_info_dict = {}

    # For each column in the dataframe, check if datatype is object. If it is,
    # get the unique values, count them, and store them in the dictionary.
    for col in df:
        if df[col].dtype == 'object':
            column_items = df[col].unique()
            cat_info_dict.update({col : {'unique_count': len(df[col].unique()),
                                         'unique_items': column_items}})
        else:
            pass

    # Create DataFrame from dictionary.
    cat_df = pd.DataFrame.from_dict(cat_info_dict, orient= 'index')

    # Merge DataFrame with categorical information with main DataFrame.
    info = info.join(cat_df,how='outer')

    # Drop useless count column
    info = info.drop(['count'], axis=1)

    # Print the dataset information.
    print('DATASET INFORMATION:')
    print('{0} rows x {1} columns'.format(len(df),len(df.columns)))
    print('--> {0} rows have missing values'.format(missing_rows_ct))
    print('--> {0} rows are duplicates'.format(dupcount))

    # If a dependent variable is provided, provide measurements against that variable.
    if dv != None:
        # If the dv is categorical, get percentages matching each category.
        if df[dv].dtype == 'object':

            print('--> Dependent Variable Values:')

            # get count of unique values in dependent variable column
            cat_list = df[dv].unique()

            # Get count of the number of unique items.
            unique_items_count = len(cat_list)

            # Calculate prpoportion for each unique dependent variable value.
            for i in range(0, unique_items_count):
                # Use i to pull each item from list of items.
                category = cat_list[i]

                # Get number of items that match that value of the dependent variable
                category_count = len(df[(df[dv] == category)])

                # For each category the percent of values where the column value matches that category.
                category_prop = category_count / len(df)

                print('    {0} = {1} : {2}'.format(dv, category, category_prop))

        # I will add code for numerical dependent variables later if the need arises.
        else:
            pass

    return info

########### CLEAN DATA, REMOVE WHITESPACE AND REMOVE UNNECESSARY COLUMNS ###############

# Remove whitespace everywhere in df.
def remove_whitespace(df):
    """Remove all whitespace from all cells and columns in the DataFrame."""
    df_obj = df.select_dtypes(['object'])
    df[df_obj.columns] = df_obj.apply(lambda x: x.str.strip())
    return df

df = remove_whitespace(df)

# Since ' ?' is being used as a placeholder, let's remove entries with ?s in them.
# Thank you stack overflow for helping with me with this line (see citation at the bottom of script)
df = df[(df != '?').all(axis=1)]

# education-num is NOT referring to number of years of schooling - it is just a number code for education column.
# Since the education column already has these values decoded, let's drop the education-num column.
df = df.drop(['education-num'], axis=1)

# Let's look at the scatter matrix first to get a sense of the big picture.
scatter_matrix(df,figsize= (10,10))

# Convert DataFrame to json.
outfile = df.to_json(path_or_buf=None, orient='records', index=True)

# Save the raw dataset to disk.
with open('adult_raw.json', 'w') as f:
    f.write(outfile)

############ LOAD DATA ################################################
# Load dataset from disk into DataFrame.
df = pd.read_json('adult_raw.json', orient='columns'
                 'records', convert_axes=True, convert_dates=True, keep_default_dates=True,
                 numpy=False, precise_float=False, date_unit=None, encoding=None)

###########  REMOVE OUTLIERS AND REVIEW DISTRIBUTIONS #######################

# See what dataset looks like with question marks and whitespace removed.
df_info(df)

# Education-num, and hours per week both look normally distributed. There are a few outliers skewing the age distribution that may also
# be concealing a normal distrubtion. Lets remove them and see what happens.

def replace_outliers(df, field, placeholder='median'):
    """Remove the outliers from a coulumn in a dataframe, using the instructions provided
       in the dictionary.
       * Note that the data atrophy that can add up if you use this function multiple times."""

    # Start values replaced counter.
    replaced_count = 0

    # Get initial length of datset before processing.
    initial_length = len(df)

     # Extract the numeric field from the dataframe, getting the column name from the dict.
    series = df[field]

    # Get upper limit by adding the mean + 2 standard deviations.
    upper_limit = np.mean(series) + 2*np.std(series)

    # Get the lower limit by subtracting two standard deviations from the mean.
    lower_limit = np.mean(series) - 2*np.std(series)

    # Flag entries with value above upper_limit and count them.
    high_outliers = df.loc[:, field] > upper_limit

    # Flag low outliers with values below lower_limit and count them.
    low_outliers = df.loc[:, field] < lower_limit

    # Valid entries is the complement of outliers
    valid_entries = ~(high_outliers | low_outliers)

    # If the user specifies to use the median, calculate the placeholder value from the median.
    if placeholder == 'median':
        repl = np.median(series)

    # If the user specifies to use the mean, use the valid entries to calculate the mean for the placeholder.
    if placeholder == 'mean':
        repl = np.mean(df.loc[valid_entries, field])

    # Replace both high and low outliers with placeholder value.
    replaced_count += df.loc[high_outliers, field].count()
    df.loc[high_outliers, field] = repl

    replaced_count += df.loc[low_outliers, field].count()
    df.loc[low_outliers, field] = repl

    replaced_portion = replaced_count/initial_length
    print('{0} outliers removed ({1})'.format(replaced_count, replaced_portion))

    # Return the DataFrame with the outliers removed.
    return df

# Run function to remove outliers from age field.
df = replace_outliers(df, 'age')

# Remove outliers from census weights field and see what happens
df = replace_outliers(df, 'fnlwgt')

plt.hist(df['fnlwgt'])
# Looks like with outliers removed, census weights are normally distributed. The data loss
# is a little less than 4% of the remaining data
plt.show()

scatter_matrix(df,figsize= (10,10))

########### CREATE Z SCALER FUNCTION (WILL RUN LATER) ################
def batch_zscaler(data, column_list):
    """ Run SKL Standard Scaler to z normalize a list of columns.
        Inputs:
        - data (a dataframe)
        - column_list (a list of strings specifying the columns to normalize)"""
    for col in column_list:
        # Now that the outliers are removed from the age column, the age column looks normally distributed.
        # Let's make a z-score column for the age attribute.
        col_values = data[col]

        # Reshape the column values array
        col_values_reshaped = col_values.values.reshape(-1,1)

        # Fit z scaler according to values in column.
        standardization_scale = StandardScaler().fit(col_values_reshaped)

        # Transform values according to z-scaler
        data[col] = standardization_scale.transform(col_values_reshaped)

    return data

########### STEP 6: BIN HOURS PER WEEK INTO CATEGORICAL VARIABLES ##################

def bin_numeric(x, bin_count): # data = data array, bounds = boundaries array
    """Take an array with the data and an array with the bin boundaries
    and then bin the data according to the boundaries supplied"""

    # Get bounds of data using minimum and maximum values of data and bin count.
    bounds = np.linspace(np.min(x), np.max(x), bin_count + 1)

    # Get number of bins
    bounds_count = len(bounds)

    # Get length of the array to be binned.
    array_length = len(x)

    # Create empty integer array to store the bin numbers (output)
    y = np.empty(array_length, int)

    # For each boundary, tag the data within each boundary, iterating
    # until all boundaries are tagged.
    for i in range(1, bounds_count):
        # If the data is greater than the bound of bin i-1 and less
        # than the upper bound of bin i, set the bin to i
        y[(x >= bounds[i-1]) & (x < bounds[i])] = i

    # If the value of x is right on the borderline of one of the bounds, bin it in the bound below it.
    y[x == bounds[-1]] = bounds_count - 1

    # Return the array of the bounds correspoding to each value in the array.
    return y

# Pull the data from the column into a variable.
hours = df['hours-per-week']
hours_vals = hours.values

# The hours per week range from 0 - 100. Let's bin it into 9 bins.
x_binned = bin_numeric(hours_vals, 9)

# Store the hours category in a new column in the DataFrame.
df['hours_cat'] = x_binned

# Rename categories to reflect approximate boundaries. Note that these boundaries are not exact since the boundaries are not exact integers.
df.loc[df.loc[:, "hours_cat"] == 1, "hours_cat"] = "works-0-11h"
df.loc[df.loc[:, "hours_cat"] == 2, "hours_cat"] = "works-12-22h"
df.loc[df.loc[:, "hours_cat"] == 3, "hours_cat"] = "works-23-34h"
df.loc[df.loc[:, "hours_cat"] == 4, "hours_cat"] = "works-34-45h"
df.loc[df.loc[:, "hours_cat"] == 5, "hours_cat"] = "works-45-55h"
df.loc[df.loc[:, "hours_cat"] == 6, "hours_cat"] = "works-55-66h"
df.loc[df.loc[:, "hours_cat"] == 7, "hours_cat"] = "works-66-77h"
df.loc[df.loc[:, "hours_cat"] == 8, "hours_cat"] = "works-77-88h"
df.loc[df.loc[:, "hours_cat"] == 9, "hours_cat"] = "works-88-100h"

############ STEP 7: PROCESS ADDITIONAL CATEGORICAL VARIABLES AND CREATE DUMMY VARIABLES

# Judging from the info output, many of the missing fields in workclass are also missing occupation fields.
# Since I anticipate these fields are going to be very important for the model, let's drop the missing entries in workclass
df = df[pd.notnull(df['occupation'])]

# Looks like removing this field removed all but about 500 the missing entries, which is convenient.
# Since we have a dense dataset, let's take a closer look at these missing entries.
missing = df[pd.isnull(df['native-country'])]

# With an excption of some outliers in the capital-gains and capital-losses attributes, these values appear to be similarly distributed
# to the population. These account for around 2% of the dataset.

# If I worked at the census, I would ask a domain expert what their
# sense of who these people might be. Let's say I was told they are
# mostly Americans.
df.loc[df.loc[:, "native-country"].isna(), "native-country"] = ' United-States'

# Create column categorizing whether indidivual is employed by the government.
govt_cats = ["Federal-gov", 'Local-gov', 'State-gov']
df.loc[df['workclass'].isin(govt_cats), 'is_govt_emp'] = 1
df.loc[~df['workclass'].isin(govt_cats), 'is_govt_emp'] = 0

# Create column categorizing whether indidivual is self-employed.
self_emp_cats = ['Self-emp-not-inc', 'Self-emp-inc']
df.loc[df['workclass'].isin(self_emp_cats), 'self_emp'] = 1
df.loc[~df['workclass'].isin(self_emp_cats), 'self_emp'] = 0

# Since there are tons of different values for native country, this might slow down the algorhythum, so lets encode the variable
# into a boolean column for whether the individal is an immigrant.
df["is-immigrant"] = (df['native-country'] != " United-States").astype(int)

# Let's drop country of origin. Since we have a saved copy we can bring it back later if necessary.
df = df.drop(['native-country'],axis=1)

# View the dataset after changes were made.
info = df_info(df, 'income_cat')
print(info)

# Select columns to encode with dummies.
# Format is 'column_name' : drop?
dummycols = {'sex' : True,
               'income_cat': True,
                'hours_cat' : True,
                'education' : True,
                'marital-status' : True,
                'occupation' : True,
                'race' : True,
                'relationship' : True,
                'workclass' : True}

def make_dummies(df, colinfo):
    """Use instructions from user-supplied dictionary to create category
       dummy columns in a dataframe."""
    for key, value in colinfo.items():

        # Get column name from dict.
        colname = key

        # Get boolean instruction on whether to drop obsolete columns from dict.
        drop = value

        # Get list of unique items in the selected column.
        unique_items = df[colname].unique()

        # Get count of the number of unique items.
        unique_items_count = len(df[colname].unique())

        # For all but 1 of the unique items,
        for i in range(1, unique_items_count):
            # Use i to pull each item from list of items.
            item = unique_items[i]

            # Remove spaces from item string so that column name is neater.
            item.strip()

            # Generate column name from item name and build column with boolean value.
            df.loc[:, "is_{0}".format(item)] = (df.loc[:, colname] == item).astype(int)

        # If user wants to drop the obsolete column, drop it.
        if drop == True:
            df = df.drop([colname],axis=1)

        # If user does not want to drop obsolete column, pass.
        if drop == False:
            pass

    # Return modified DataFrame
    return df

df2 = make_dummies(df, dummycols)

wk = df['workclass'].values
print(wk)

# Let's see what the DataFrame looks like now.
print(df2[:5])

info2 = df_info(df2)

print(df2.columns)

## SAVE DATASET TO DISK
df2.to_csv('Berkowitz-M02-Dataset.csv', header = True)

################ DEFINE HELPER FUNCTIONS FOR RESULTS ANALYSIS ###############

def feature_importance(model, X_train):
    """Plot importances for each feature in a given model.
    Inputs: Model and Training Data
    Outputs: Plotted feature importances"""

    # Pull feature labels from the training data.
    feat_labels = X_train.columns

    # Pull importances array from the selected model object.
    importances = model.feature_importances_

    ## PLOT IMPORTANCES

    # Sort the importances, largest to smallest.
    indices = np.argsort(importances)[::-1]

    # Set figure size.
    figure(num=None, figsize=(15, 6), dpi=80, facecolor='w', edgecolor='k')

    # Set plot title.
    plt.title('Feature Importance')

    # Set bar plot.
    plt.bar(range(X_train.shape[1]),
            importances[indices],
            align='center')

    # Set X ticks.
    plt.xticks(range(X_train.shape[1]),
               feat_labels[indices], rotation=90)

    # Set X limit.
    plt.xlim([-1, X_train.shape[1]])

    # Show plot.
    plt.show()

def make_confmat(y_true, y_pred):
    """Calculate and graph a confusion matrix based on arrays with
       true and predicted values."""
    confmat = confusion_matrix(y_true=y_true, y_pred=y_pred)

    fig, ax = plt.subplots(figsize=(2.5, 2.5))
    ax.matshow(confmat, cmap=plt.cm.Blues, alpha=0.3)
    for i in range(confmat.shape[0]):
        for j in range(confmat.shape[1]):
            ax.text(x=j, y=i, s=confmat[i, j], va='center', ha='center')

    plt.xlabel('Predicted label')
    plt.ylabel('True label')

    plt.tight_layout()
    #plt.savefig('images/06_09.png', dpi=300)
    plt.show()

    # Show detailed statistics

    # Generate accuracy score from arrays of target values and actual values.
    AR = accuracy_score(y_true, y_pred)
    print("\nAccuracy rate:", AR)

    # Calculate error rate by subtracting accuracy score from 1.
    ER = 1.0 - AR
    print("\nError rate:", ER)

    # Calculate precision score from target array and values array.
    P = precision_score(y_true, y_pred)
    print ("\nPrecision:", np.round(P, 2))

    # Calculate recall score from target array and values array.
    R = recall_score(y_true, y_pred)
    print ("\nRecall:", np.round(R, 2))

    # Calculate F1 score from target array and values array.
    F1 = f1_score(y_true, y_pred)
    print ("\nF1 score:", np.round(F1, 2))

def plot_auc(targets, predictions, LW = 1.5, LL = 'lower right', LC = 'darkgreen'):
    """Plot AUC to show model preformance.
    Inputs:
      targets = 1D Numpy array with prediction targets.
      predictions = 1D Numpy array with predictions.
      Variables customizing graph appearance (optional)
    Outputs:
      True Positive Rates (TPR)
      AUC calculation.
      Plot of the ROC curve.
      """
    LW = LW # line width for plots
    LL = LL # legend location
    LC = LC # Line Color

    fpr, tpr, th = roc_curve(targets, predictions) # False Positive Rate, True Posisive Rate, probability thresholds

    # Calculate AUC score.
    AUC = auc(fpr, tpr)

    # Display calculated scores, rounded to the second digit.
    print("\nTP rates:", np.round(tpr, 2))
    print("\nFP rates:", np.round(fpr, 2))
    print("\nProbability thresholds:", np.round(th, 2))

    # Generate figure for ROC curve.
    plt.figure()
    plt.title('Receiver Operating Characteristic curve')

    # Set limits to the x and y ranges to give the plot some breathing room.
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])

    plt.xlabel('FALSE Positive Rate')
    plt.ylabel('TRUE Positive Rate')

    # Plot False Positive Rate against True Positive Rate.
    plt.plot(fpr, tpr, color=LC,lw=LW, label='ROC curve (area = %0.2f)' % AUC)

    # Reference line for random classifier
    plt.plot([0, 1], [0, 1], color='navy', lw=LW, linestyle='--') # reference line for random classifier
    plt.legend(loc=LL)
    plt.show()

    # Show rounded AUC scores using auc and roc_auc functions.
    print("\nAUC score (using auc function):", np.round(AUC, 2))
    print("\nAUC score (using roc_auc_score function):", np.round(roc_auc_score(targets, predictions), 2), "\n")

###################################################################################

############ LOAD THE DATA ##############################
df = pd.read_csv('Berkowitz-M02-Dataset.csv')

print(df.columns)

############ SPLIT AND SCALE THE DATA ###################

## NOTE: I'm not going to normalize the age column since they have already been normalized.
# If I were to normalize them, however, heres the code I would use.
# # Transform values according to z-scaler and put them in new column.
# df['znorm-age'] = standardization_scale.transform(ages_reshaped)
####

# Use list to specify columns to ignore. I'm removing the unnamed column since that's just duplicated indexes and the is 50k column
# Since that is the target variable. Also removing hours-per-week since thats already been binned.
exclude_cols = ['is_>50K', 'Unnamed: 0', 'hours-per-week']

# Set y to the target variable.
y = df["is_>50K"]

# Set X to the dataset, with the excluded columns excluded.
X = df.loc[:, ~df.columns.isin(exclude_cols)]

# Extract features names from column titles.
features = X.columns

# Split data into training and test data using 40 percent for testing.
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.40, random_state=42)

# Run the Z scaler on the training data.
X_train = batch_zscaler(X_train, ['age', 'fnlwgt'])

# Run the Z scaler on the testing data.
X_test = batch_zscaler(X_test, ['age', 'fnlwgt'])

############# LOAD DATA AND RUN RANDOM FOREST MODEL ##########
# I chose the Random Forest for several reasons. It's easy to use
# and generally provides strong classification performance
# with a relatively low risk of overfitting.

estimators = 500 # number of trees parameter
mss = 3 # mininum samples split parameter

# Setup random forest classifier
forest = sklearn.ensemble.RandomForestClassifier(n_estimators=estimators, min_samples_split=mss)

# Train on classifier
forest.fit(X_train,y_train)

# Generate prediction on the test data
forest_pred = forest.predict(X_test)

# I'm starting to wonder if it really is a good idea to drop the census weights.
# Get feature importance for random forest and graph them.

feature_importance(forest, X_train)

# Graph Confusion Matrix
make_confmat(y_test, forest_pred)

# Run ROC plotting function.
plot_auc(y_test, forest_pred)

# Create dataframe showing predicted values vs. actual values
pred_df = pd.DataFrame(y_test)
pred_df['prediction'] = forest_pred

# The Random forest got 85 % accuracy.
# Though the ratio of Postives to Negatives is about right, some classification errors were made.
# About 2/3ds of the classification errors were False Negatives. This contributes to a precision score
# of 73% that is 12% below the overall model accuracy.
# The starting F1 score is .67, not great but it's a start.
# I would consider just removing the census weights, but the Feature Importance appears relatively
# high for the census weights.
# AUC is .78, showing room for improvement.

########## RUN SUPPORT VECTOR MACHINES MODEL ########
# I chose the SVM because I suspect that the different
# attributes of this dataset could be interrelated in ways
# that are otherwise linearly inseperable.

svm_clf = svm.SVC()

# Train on classifier
svm_clf.fit(X_train,y_train)

# Generate prediction on the test data
svm_pred = svm_clf.predict(X_test)

# Show model results analysis
plot_auc(y_test, svm_pred)
make_confmat(y_test, svm_pred)

# SVM classifier with default settings results in 87% accuracy.
# Shows signicant imporvement in precision, contributing to better overall accuracy.
# The model has low Reliability when predicting positive class instances, as it only discovered
# slightly more than half of the cases where incomes were over 50k.
# Though the SVM classifier declared roughly the same amount of false negatives as the
# Random Forest, it was significantly better at avoiding false positives.
# The F1 score is also better at .7.
# The AUC score is unchanged, but the ROC curve does show a very slight improvement.

##### Let's reload the data and see how the models do without the 'age' column binned.

############ LOAD THE DATA ##############################
df = pd.read_csv('Berkowitz-M02-Dataset.csv')

print(df.columns)

####### RERUN MODELS WITHOUT CATEGORICAL VARIABLES ############
# Use list to specify columns to ignore. I'm removing the unnamed column since that's
# just duplicated indexes and the 50k column is the target variable.
# Also the hours-per-week bins to see how that affects performance.
exclude_cols = ['is_>50K', 'Unnamed: 0', 'is_works-34-45h', 'is_works-45-55h',
       'is_works-77-88h', 'is_works-23-34h', 'is_works-55-66h',
       'is_works-66-77h', 'is_works-0-11h', 'is_works-88-100h']

# Set y to the target variable.
y = df["is_>50K"]

# Set X to the dataset, with the excluded columns excluded.
X = df.loc[:, ~df.columns.isin(exclude_cols)]

############ SPLIT AND SCALE THE DATA ###################

# Extract features names from column titles.
features = X.columns

# Split data into training and test data using 40 percent for testing.
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.40, random_state=42)

# Run the Z scaler on the training data.
X_train = batch_zscaler(X_train, ['age', 'fnlwgt'])

# Run the Z scaler on the testing data.
X_test = batch_zscaler(X_test, ['age', 'fnlwgt'])

############# LOAD DATA AND RUN RANDOM FOREST MODEL ##########
estimators = 500 # number of trees parameter
mss = 3 # mininum samples split parameter

# Setup random forest classifier
forest = sklearn.ensemble.RandomForestClassifier(n_estimators=estimators, min_samples_split=mss)

# Train on classifier
forest.fit(X_train,y_train)

# Generate prediction on the test data
forest_pred = forest.predict(X_test)

# Get feature importance for random forest and graph them.
feature_importance(forest, X_train)

# Graph Confusion Matrix
make_confmat(y_test, forest_pred)

# Run ROC plotting function.
plot_auc(y_test, forest_pred)

# Create dataframe showing predicted values vs. actual values
pred_df = pd.DataFrame(y_test)
pred_df['prediction'] = forest_pred

# The Random Forest model performs the roughly same in the dataset
# without the ages binned as it does when the ages are binned.
# The F1 score is about the same and the confusion matrix counts,
# Precision and Recall are also largely unaffected by the bins.
# AUC is .78, not showing a significant impact with the bins removed.

########## RUN SUPPORT VECTOR MACHINES MODEL #####
svm_clf = svm.SVC()

# Train on classifier
svm_clf.fit(X_train,y_train)

# Generate prediction on the test data
svm_pred = svm_clf.predict(X_test)

# Show model results analysis
plot_auc(y_test, svm_pred)
make_confmat(y_test, svm_pred)

# Though the SVM model performed relatively well with the age-binned dataset,
# the lack of bins has resulted in a much higher rate of false negatives as well
# as false positives. The F1 score of .66 is the worst of the four model/dataset
# variants. AUC has also shown a slight negative impact, shrinking to .75.
# This is reflected by a slight change in the curve.

############## SUMMARY ##############################################
# Cleaning and outlier removal was performed on the Census Dataset.
# Two classifiers were tested, the Random Forest and SVM.
# Each classifier was run on one dataset with the 'age' column binned
# and one dataset in which the the 'age' column was numeric to test
# the impact of the bins on each classifiers' performance.
# The performance of the random forest model satisfactory, showing
# 85% accuracy with both binned and non-binned datasets.
# On the other hand, the SVM surpassed both Random Forest Classifiers
# when the bins were added.

# The SVM classifier showed the strongest performance and best fit
# of all the models tested here when run on the binned dataset.
# This is in contrast to the Random Forest, which did not appear
# to be impacted by bins. This suggests that the SVM could be a very
# practical tool for datasets where significant feature engineering
# has been performed.

# CODE REFERENCES:
# Dictionary processing function adapted from:
# Title: How to get the current time in Python
# Author: TigerhawkT3
# Date: 11/26/2017
# Code version: N/A
# Availability: https://stackoverflow.com/a/40823642

# Flagging values in mulitple columns (answer to StackOverflow question I asked)
# Title: Using df.loc to flag values in multiple columns at once
# Author: coldspeed
# Date: 2/6/2018
# Code version: N/A
# Availability: https://stackoverflow.com/a/54545338

# Binning function adapted from:
# Title: L06-A-1 Normalizing and Binning in Python
# Author: Zacharias Voulgaris
# Date: NA
# Code version: N/A
# Availability: Not publicly available.

# Feature importance/Confusion matrix function adapted from:
# Title: Python Machine Learning 2nd ed.
# Author: Sebastian Rashka
# Date: 11/19/2018
# Available from: https://github.com/rasbt/python-machine-learning-book-2nd-edition/blob/master/code/ch04/ch04.ipynb

# Confusion Matrix/ROC/AUC code adapted from:
# Title: L09-AccuracyMeasures.py
# Author: Zacharias Voulgaris
# Date: NA
# Code version: N/A
# Availability: Not publicly available.


from django.db import models

from django.contrib.postgres.fields import ArrayField
from django.contrib.postgres.search import SearchVector, SearchVectorField

from regcore.models import Part

class SearchIndex(models.Model):
    type = models.CharField(max_length=32)
    label = ArrayField(base_field=models.CharField(max_length=32))
    content = models.TextField()
    parent = models.JSONField(null=True)
    part = models.ForeignKey(Part, on_delete=models.CASCADE)
    search_vector = SearchVectorField()

    class Meta:
        unique_together = ['label', 'part']

def create_search(part, piece, memo, parent=None, ):
    if piece.get("node_type", None) == "SECTION":
        si = SearchIndex(
            label = piece["label"],
            part = part,
            parent = piece,
            type = piece["node_type"],
            content = piece.get("title", piece.get("text", "")),
        )
        children = piece.pop("children", []) or []
        for child in children:
            si.content = si.content + child.get("text", "")
        memo.append(si)
    else:
        children = piece.pop("children", []) or []
        for child in children:
            create_search(part, child, memo, parent=piece)

    return memo

def update_search(sender, instance, created, **kwargs):
    SearchIndex.objects.filter(part=instance).delete()
    contexts = create_search(instance, instance.document, [])
    SearchIndex.objects.bulk_create(contexts, ignore_conflicts=True)
    SearchIndex.objects.filter(part=instance).update(search_vector=SearchVector('content'))