"""
Tests of Tax-Calculator using puf.csv input.

Note that the puf.csv file that is required to run this program has
been constructed by the Tax-Calculator development team by merging
information from the most recent publicly available IRS SOI PUF file
and from the Census CPS file for the corresponding year.  If you have
acquired from IRS the most recent SOI PUF file and want to execute
this program, contact the Tax-Calculator development team to discuss
your options.

Read Tax-Calculator/TESTING.md for details.
"""
# CODING-STYLE CHECKS:
# pycodestyle test_pufcsv.py
# pylint --disable=locally-disabled test_pufcsv.py

import os
import json
import pytest
import numpy as np
import pandas as pd
# pylint: disable=import-error
from taxcalc import Policy, Records, Calculator

START_YEAR = 2017

@pytest.mark.requires_pufcsv
def test_agg(tests_path, puf_fullsample):
    """
    Test Tax-Calculator aggregate taxes with no policy reform using
    the full-sample puf.csv and a small sub-sample of puf.csv
    """
    # pylint: disable=too-many-locals,too-many-statements
    nyrs = 10
    # create a baseline Policy object with current-law policy parameters
    baseline_policy = Policy()
    # create a Records object (rec) containing all puf.csv input records
    recs = Records(data=puf_fullsample)
    # create a Calculator object using baseline policy and puf records
    calc = Calculator(policy=baseline_policy, records=recs)
    calc.advance_to_year(START_YEAR)
    calc_start_year = calc.current_year
    # create aggregate diagnostic table (adt) as a Pandas DataFrame object
    adt = calc.diagnostic_table(nyrs).round(1)  # column labels are int
    taxes_fullsample = adt.loc["Combined Liability ($b)"]
    # compare actual DataFrame, adt, with the expected DataFrame, edt
    aggres_path = os.path.join(tests_path, 'pufcsv_agg_expect.csv')
    edt = pd.read_csv(aggres_path, index_col=False)  # column labels are str
    edt.drop('Unnamed: 0', axis='columns', inplace=True)
    assert len(adt.columns.values) == len(edt.columns.values)
    diffs = False
    for icol in adt.columns.values:
        if not np.allclose(adt[icol].values, edt[str(icol)].values):
            diffs = True
    if diffs:
        new_filename = '{}{}'.format(aggres_path[:-10], 'actual.csv')
        adt.to_csv(new_filename, float_format='%.1f')
        msg = 'PUFCSV AGG RESULTS DIFFER FOR FULL-SAMPLE\n'
        msg += '-------------------------------------------------\n'
        msg += '--- NEW RESULTS IN pufcsv_agg_actual.csv FILE ---\n'
        msg += '--- if new OK, copy pufcsv_agg_actual.csv to  ---\n'
        msg += '---                 pufcsv_agg_expect.csv     ---\n'
        msg += '---            and rerun test.                ---\n'
        msg += '---       (both are in taxcalc/tests)         ---\n'
        msg += '-------------------------------------------------\n'
        raise ValueError(msg)
    # create aggregate diagnostic table using unweighted sub-sample of records
    fullsample = puf_fullsample
    rn_seed = 2222  # to ensure sub-sample is always the same
    subfrac = 0.05  # sub-sample fraction
    subsample = fullsample.sample(frac=subfrac, random_state=rn_seed)
    recs_subsample = Records(data=subsample)
    calc_subsample = Calculator(policy=baseline_policy, records=recs_subsample)
    calc_subsample.advance_to_year(START_YEAR)
    adt_subsample = calc_subsample.diagnostic_table(nyrs)
    # compare combined tax liability from full and sub samples for each year
    taxes_subsample = adt_subsample.loc["Combined Liability ($b)"]
    msg = ''
    for cyr in range(calc_start_year, calc_start_year + nyrs):
        reltol = 0.031  # maximum allowed relative difference in tax liability
        if not np.allclose(taxes_subsample[cyr], taxes_fullsample[cyr],
                           atol=0.0, rtol=reltol):
            reldiff = (taxes_subsample[cyr] / taxes_fullsample[cyr]) - 1.
            line1 = '\nPUFCSV AGG SUB-vs-FULL RESULTS DIFFER IN {}'
            line2 = '\n  when subfrac={:.3f}, rtol={:.4f}, seed={}'
            line3 = '\n  with sub={:.3f}, full={:.3f}, rdiff={:.4f}'
            msg += line1.format(cyr)
            msg += line2.format(subfrac, reltol, rn_seed)
            msg += line3.format(taxes_subsample[cyr],
                                taxes_fullsample[cyr],
                                reldiff)
    if msg:
        raise ValueError(msg)

MTR_TAX_YEAR = 2013
MTR_NEG_DIFF = False  # set True to subtract (rather than add) small amount
# specify payrolltax mtr histogram bin boundaries (or edges):
PTAX_MTR_BIN_EDGES = [0.0, 0.02, 0.04, 0.06, 0.08,
                      0.10, 0.12, 0.14, 0.16, 0.18, 1.0]
#        the bin boundaries above are arbitrary, so users
#        may want to experiment with alternative boundaries
# specify incometax mtr histogram bin boundaries (or edges):
ITAX_MTR_BIN_EDGES = [-1.0, -0.30, -0.20, -0.10, 0.0,
                      0.10, 0.20, 0.30, 0.40, 0.50, 1.0]
#        the bin boundaries above are arbitrary, so users
#        may want to experiment with alternative boundaries

def mtr_bin_counts(mtr_data, bin_edges, recid):
    """
    Compute mtr histogram bin counts and return results as a string.
    """
    res = ''
    (bincount, _) = np.histogram(mtr_data.round(decimals=4), bins=bin_edges)
    sum_bincount = np.sum(bincount)
    res += '{} :'.format(sum_bincount)
    for idx in range(len(bin_edges) - 1):
        res += ' {:6d}'.format(bincount[idx])
    res += '\n'
    if sum_bincount < mtr_data.size:
        res += 'WARNING: sum of bin counts is too low\n'
        recinfo = '         mtr={:.2f} for recid={}\n'
        mtr_min = mtr_data.min()
        mtr_max = mtr_data.max()
        bin_min = min(bin_edges)
        bin_max = max(bin_edges)
        if mtr_min < bin_min:
            res += '         min(mtr)={:.2f}\n'.format(mtr_min)
            for idx in range(mtr_data.size):
                if mtr_data[idx] < bin_min:
                    res += recinfo.format(mtr_data[idx], recid[idx])
        if mtr_max > bin_max:
            res += '         max(mtr)={:.2f}\n'.format(mtr_max)
            for idx in range(mtr_data.size):
                if mtr_data[idx] > bin_max:
                    res += recinfo.format(mtr_data[idx], recid[idx])
    return res

def nonsmall_diffs(linelist1, linelist2, small=0.0):
    """
    Return True if line lists differ significantly; otherwise return False.
    Significant numerical difference means one or more numbers differ (between
    linelist1 and linelist2) by more than the specified small amount.
    """
    # embedded function used only in nonsmall_diffs function
    def isfloat(value):
        """
        Return True if value can be cast to float; otherwise return False.
        """
        try:
            float(value)
            return True
        except ValueError:
            return False
    # begin nonsmall_diffs logic
    assert isinstance(linelist1, list)
    assert isinstance(linelist2, list)
    if len(linelist1) != len(linelist2):
        return True
    assert 0.0 <= small <= 1.0
    epsilon = 1e-6
    smallamt = small + epsilon
    for line1, line2 in zip(linelist1, linelist2):
        if line1 == line2:
            continue
        else:
            tokens1 = line1.replace(',', '').split()
            tokens2 = line2.replace(',', '').split()
            for tok1, tok2 in zip(tokens1, tokens2):
                tok1_isfloat = isfloat(tok1)
                tok2_isfloat = isfloat(tok2)
                if tok1_isfloat and tok2_isfloat:
                    if abs(float(tok1) - float(tok2)) <= smallamt:
                        continue
                    else:
                        return True
                elif not tok1_isfloat and not tok2_isfloat:
                    if tok1 == tok2:
                        continue
                    else:
                        return True
                else:
                    return True
        return False

@pytest.mark.requires_pufcsv
def test_mtr(tests_path, puf_path):
    """
    Test Tax-Calculator marginal tax rates with no policy reform using puf.csv

    Compute histograms for each marginal tax rate income type using
    sample input from the puf.csv file and writing output to a string,
    which is then compared for differences with EXPECTED_MTR_RESULTS.
    """
    # pylint: disable=too-many-locals,too-many-statements
    assert len(PTAX_MTR_BIN_EDGES) == len(ITAX_MTR_BIN_EDGES)
    # construct actual results string, res
    res = ''
    if MTR_NEG_DIFF:
        res += 'MTR computed using NEGATIVE finite_diff '
    else:
        res += 'MTR computed using POSITIVE finite_diff '
    res += 'for tax year {}\n'.format(MTR_TAX_YEAR)
    # create a Policy object (clp) containing current-law policy parameters
    clp = Policy()
    clp.set_year(MTR_TAX_YEAR)
    # create a Records object (puf) containing puf.csv input records
    puf = Records(data=puf_path)
    recid = puf.RECID  # pylint: disable=no-member
    # create a Calculator object using clp policy and puf records
    calc = Calculator(policy=clp, records=puf)
    res += '{} = {}\n'.format('Total number of data records', puf.array_length)
    res += 'PTAX mtr histogram bin edges:\n'
    res += '     {}\n'.format(PTAX_MTR_BIN_EDGES)
    res += 'ITAX mtr histogram bin edges:\n'
    res += '     {}\n'.format(ITAX_MTR_BIN_EDGES)
    variable_header = 'PTAX and ITAX mtr histogram bin counts for'
    # compute marginal tax rate (mtr) histograms for each mtr variable
    for var_str in Calculator.MTR_VALID_VARIABLES:
        zero_out = (var_str == 'e01400')
        (mtr_ptax, mtr_itax, _) = calc.mtr(variable_str=var_str,
                                           negative_finite_diff=MTR_NEG_DIFF,
                                           zero_out_calculated_vars=zero_out,
                                           wrt_full_compensation=False)
        if zero_out:
            # check that calculated variables are consistent
            assert np.allclose((calc.array('iitax') +
                                calc.array('payrolltax')),
                               calc.array('combined'))
            assert np.allclose((calc.array('ptax_was') +
                                calc.array('setax') +
                                calc.array('ptax_amc')),
                               calc.array('payrolltax'))
            assert np.allclose(calc.array('c21060') - calc.array('c21040'),
                               calc.array('c04470'))
            assert np.allclose(calc.array('taxbc') + calc.array('c09600'),
                               calc.array('c05800'))
            assert np.allclose((calc.array('c05800') +
                                calc.array('othertaxes') -
                                calc.array('c07100')),
                               calc.array('c09200'))
            assert np.allclose(calc.array('c09200') - calc.array('refund'),
                               calc.array('iitax'))
        if var_str == 'e00200s':
            # only MARS==2 filing units have valid MTR values
            mtr_ptax = mtr_ptax[calc.array('MARS') == 2]
            mtr_itax = mtr_itax[calc.array('MARS') == 2]
        res += '{} {}:\n'.format(variable_header, var_str)
        res += mtr_bin_counts(mtr_ptax, PTAX_MTR_BIN_EDGES, recid)
        res += mtr_bin_counts(mtr_itax, ITAX_MTR_BIN_EDGES, recid)
    # check for differences between actual and expected results
    mtrres_path = os.path.join(tests_path, 'pufcsv_mtr_expect.txt')
    with open(mtrres_path, 'r') as expected_file:
        txt = expected_file.read()
    expected_results = txt.rstrip('\n\t ') + '\n'  # cleanup end of file txt
    if nonsmall_diffs(res.splitlines(True), expected_results.splitlines(True)):
        new_filename = '{}{}'.format(mtrres_path[:-10], 'actual.txt')
        with open(new_filename, 'w') as new_file:
            new_file.write(res)
        msg = 'PUFCSV MTR RESULTS DIFFER\n'
        msg += '-------------------------------------------------\n'
        msg += '--- NEW RESULTS IN pufcsv_mtr_actual.txt FILE ---\n'
        msg += '--- if new OK, copy pufcsv_mtr_actual.txt to  ---\n'
        msg += '---                 pufcsv_mtr_expect.txt     ---\n'
        msg += '---            and rerun test.                ---\n'
        msg += '-------------------------------------------------\n'
        raise ValueError(msg)

@pytest.mark.requires_pufcsv
def test_mtr_pt_active(puf_subsample):
    """
    Test whether including wages in active income causes
    MTRs on e00900p and e26270 to be less than -1 (i.e., -100%)
    """
    # pylint: disable=too-many-locals
    rec = Records(data=puf_subsample)
    reform_year = 2018
    # create current-law Calculator object, calc1
    pol1 = Policy()
    calc1 = Calculator(policy=pol1, records=rec)
    calc1.advance_to_year(reform_year)
    calc1.calc_all()
    mtr1_e00900p = calc1.mtr('e00900p')[2]
    mtr1_e26270 = calc1.mtr('e26270')[2]
    assert min(mtr1_e00900p) > -1
    assert min(mtr1_e26270) > -1
    # change PT rates, calc2
    reform2 = {'PT_rt7': {reform_year: 0.35}}
    pol2 = Policy()
    pol2.implement_reform(reform2)
    calc2 = Calculator(policy=pol2, records=rec)
    calc2.advance_to_year(reform_year)
    calc2.calc_all()
    mtr2_e00900p = calc2.mtr('e00900p')[2]
    mtr2_e26270 = calc2.mtr('e26270')[2]
    assert min(mtr2_e00900p) > -1
    assert min(mtr2_e26270) > -1
    # change PT_wages_active_income
    reform3 = {'PT_wages_active_income': {reform_year: True}}
    pol3 = Policy()
    pol3.implement_reform(reform3)
    calc3 = Calculator(policy=pol3, records=rec)
    calc3.advance_to_year(reform_year)
    calc3.calc_all()
    mtr3_e00900p = calc3.mtr('e00900p')[2]
    mtr3_e26270 = calc3.mtr('e26270')[2]
    assert min(mtr3_e00900p) > -1
    assert min(mtr3_e26270) > -1
    # change PT rates and PT_wages_active_income
    reform4 = {
        'PT_wages_active_income': {reform_year: True},
        'PT_rt7': {reform_year: 0.35}
    }
    pol4 = Policy()
    pol4.implement_reform(reform4)
    calc4 = Calculator(policy=pol4, records=rec)
    calc4.advance_to_year(reform_year)
    calc4.calc_all()
    mtr4_e00900p = calc4.mtr('e00900p')[2]
    mtr4_e26270 = calc4.mtr('e26270')[2]
    assert min(mtr4_e00900p) > -1
    assert min(mtr4_e26270) > -1

@pytest.mark.requires_pufcsv
def test_credit_reforms(puf_subsample):
    """
    Test personal credit reforms using puf.csv subsample
    """
    rec = Records(data=puf_subsample)
    reform_year = 2017
    # create current-law Calculator object, calc1
    pol = Policy()
    calc1 = Calculator(policy=pol, records=rec)
    calc1.advance_to_year(reform_year)
    calc1.calc_all()
    itax1 = calc1.weighted_total('iitax')
    # create personal-refundable-credit-reform Calculator object, calc2
    reform = {'II_credit': {reform_year: [1000, 1000, 1000, 1000, 1000]}}
    pol.implement_reform(reform)
    calc2 = Calculator(policy=pol, records=rec)
    calc2.advance_to_year(reform_year)
    calc2.calc_all()
    itax2 = calc2.weighted_total('iitax')
    # create personal-nonrefundable-credit-reform Calculator object, calc3
    reform = {'II_credit_nr': {reform_year: [1000, 1000, 1000, 1000, 1000]}}
    pol = Policy()
    pol.implement_reform(reform)
    calc3 = Calculator(policy=pol, records=rec)
    calc3.advance_to_year(reform_year)
    calc3.calc_all()
    itax3 = calc3.weighted_total('iitax')
    # check income tax revenues generated by the three Calculator objects
    assert itax2 < itax1  # because refundable credits lower revenues
    assert itax3 > itax2  # because nonrefundable credits lower revenues less
    assert itax3 < itax1  # because nonrefundable credits lower revenues some

@pytest.mark.requires_pufcsv
def test_puf_availability(tests_path, puf_path):
    """
    Cross-check records_variables.json data with variables in puf.csv file
    """
    # make set of variable names in puf.csv file
    pufdf = pd.read_csv(puf_path)
    pufvars = set(list(pufdf))
    # make set of variable names that are marked as puf.csv available
    rvpath = os.path.join(tests_path, '..', 'records_variables.json')
    with open(rvpath, 'r') as rvfile:
        rvdict = json.load(rvfile)
    recvars = set()
    for vname, vdict in rvdict['read'].items():
        if 'taxdata_puf' in vdict.get('availability', ''):
            recvars.add(vname)
    # check that pufvars and recvars sets are the same
    assert (pufvars - recvars) == set()
    assert (recvars - pufvars) == set()


from __future__ import print_function, division, absolute_import

import os
import stat
import textwrap

from ..cmdline import NumpydocClassCommand, argument
from ..io import TemplateProject

def chmod_plus_x(fn):
    st = os.stat(fn)
    os.chmod(fn, st.st_mode | stat.S_IEXEC)

class TemplateProjectCommand(NumpydocClassCommand):
    _group = '0-Support'
    _concrete = True
    description = __doc__
    klass = TemplateProject

    disclaimer = argument('--disclaimer', default=False, action='store_true',
                          help="Print a disclaimer about using these templates.")

    def print_disclaimer(self):
        print('\n'.join(textwrap.wrap(
            "This writes a bunch of Python files that can guide you "
            "through analyzing a system with MSMBuilder. I implore you to "
            "look at the scripts before you start blindly running them. "
            "You will likely have to change some (hyper-)parameters or "
            "filenames to match your particular project."
        )))
        print()
        print('\n'.join(textwrap.wrap(
            "More than that, however, it is important that you understand "
            "exactly what the scripts are doing. Each protein system is "
            "different, and it is up to you (the researcher) to hone in on "
            "interesting aspects. This very generic pipeline may not give "
            "you any new insight for anything but the simplest systems."
        )))

    def start(self):
        if self.disclaimer:
            self.print_disclaimer()
            print()
            print("Run again without --disclaimer to actually write tempaltes.")
            return

        self.instance.do()


"""Added conditions and setpoints

Revision ID: 12d1737b49e9
Revises: dd0acf882385
Create Date: 2019-11-26 13:41:35.771547

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql

# revision identifiers, used by Alembic.
revision = '12d1737b49e9'
down_revision = 'dd0acf882385'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('setpoints',
    sa.Column('uuid', postgresql.UUID(as_uuid=True), server_default=sa.text('uuid_generate_v4()'), nullable=False),
    sa.Column('name', sa.String(length=64), nullable=True),
    sa.Column('value', sa.String(length=128), nullable=False),
    sa.Column('type', sa.String(length=64), nullable=False),
    sa.PrimaryKeyConstraint('uuid'),
    sa.UniqueConstraint('uuid')
    )
    op.create_table('conditions',
    sa.Column('uuid', postgresql.UUID(as_uuid=True), server_default=sa.text('uuid_generate_v4()'), nullable=False),
    sa.Column('comment', sa.String(length=128), nullable=False),
    sa.Column('scenario_uuid', postgresql.UUID(as_uuid=True), nullable=True),
    sa.Column('condition_type', sa.Enum('IsEqual', 'IsMore', 'IsLess', 'NotEqual', 'InMode', 'NotInMode', name='conditiontypeenum'), nullable=True),
    sa.Column('target_uuid', postgresql.UUID(as_uuid=True), nullable=True),
    sa.Column('target_parameter_name', sa.String(length=64), nullable=False),
    sa.Column('setpoint_uuid', postgresql.UUID(as_uuid=True), nullable=True),
    sa.ForeignKeyConstraint(['scenario_uuid'], ['scenarios.uuid'], ),
    sa.ForeignKeyConstraint(['setpoint_uuid'], ['setpoints.uuid'], ),
    sa.PrimaryKeyConstraint('uuid'),
    sa.UniqueConstraint('uuid')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('conditions')
    op.drop_table('setpoints')
    # ### end Alembic commands ###


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Base
import ast
import json
import sys
import subprocess
import time

# Internal
from addons.config import ConfigMaster

from db.db_init.db_init_base import BaseBase, base_name, GeneralCatList, ValCategories, ValResult
from db.db_connection.db_external import ExternalConnectDB

# External
from PyQt5.QtWidgets import QMainWindow, QApplication, QGraphicsScene, QGraphicsPixmapItem, QFileDialog, QDialog, \
    QLabel, QPushButton, QVBoxLayout
from PyQt5.QtCore import Qt, QLineF, pyqtSignal, QThread, QObject, QRunnable, QEventLoop
from PyQt5.QtGui import QPixmap, QPen

import sqlalchemy.exc
from sqlalchemy.dialects.mysql import insert

class Scores(QRunnable):
    def __init__(self, db, config_path):
        super(QRunnable, self).__init__()

        self.db = db
        self.config = ConfigMaster(config_path)

    def run(self):

        abs_count = 0
        any_count = 0

        tp = 0
        tn = 0
        fp = 0
        fn = 0

        count = 0

        data_format = self.get_all_data()
        cats_count = self.get_cat_len()

        for row in data_format:
            cats_base = row[0]
            cats_pred = row[1]

            o_cats_vector = [0] * cats_count
            p_cats_vector = [0] * cats_count

            for i in range(cats_count):
                if i in cats_base:
                    o_cats_vector[i] = 1
                if i in cats_pred:
                    p_cats_vector[i] = 1

            if sorted(o_cats_vector) == sorted(p_cats_vector):
                abs_count += 1

            if any(cat in o_cats_vector for cat in p_cats_vector):
                any_count += 1

            cat_pair = zip(o_cats_vector, p_cats_vector)

            for pair in cat_pair:
                a = pair[0]
                b = pair[1]

                if a == b == 1:
                    tp += 1
                elif a == b == 0:
                    tn += 1
                elif a != b and b == 1:
                    fp += 1
                elif a != b and a == 1:
                    fn += 1

            count += 1

        pr = tp / (tp + fp)
        rc = tp / (tp + fn)

        f1 = tp / (tp + 0.5 * (fp + fn))

        absl = abs_count / count
        anya = any_count / count

        result = {'Precision': pr,
                  'Recall': rc,
                  'F1-score': f1,
                  'Absolute accuracy': absl,
                  'Any accuracy': anya}

        self.write(result)

    def write(self, result):
        for cat in result.keys():
            insert_stmt = insert(ValResult).values(
                metric=cat,
                value=result[cat],
            )

            on_duplicate_key_stmt = insert_stmt.on_duplicate_key_update(
                value=result[cat],
            )

            self.db.session.execute(on_duplicate_key_stmt)
            self.db.session.commit()

    def get_cat_len(self):
        data = self.db.get_all(GeneralCatList)
        return len(data)

    def get_all_data(self):
        data = self.db.get_all(ValCategories)

        data_format = []
        for row in data:
            data_format.append((ast.literal_eval(row['category_base']), ast.literal_eval(row['category_pred'])))

        return data_format

if __name__ == '__main__':
    config_path = r'E:\Projects\NeuralDiploma\conf\project.cfg'
    config_path_server = r'E:\Projects\NeuralDiploma\conf\server.cfg'

    db_ext = ExternalConnectDB(BaseBase, base_name, config_path)

    scr = Scores(db_ext, config_path_server)
    scr.run()


#!/usr/bin/python3
import re
import os
import sys
import hashlib
import smtplib
import logging
from datetime import datetime, timedelta
from calendar import timegm

from email.message import Message
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from wsgiref.handlers import format_date_time

import pyasn1
from pyasn1.codec.der import decoder, encoder
from pyasn1.type.useful import GeneralizedTime
import pyasn1_modules.pem as pem
import pyasn1_modules.rfc2459 as x509

CONFIG_TIMEDELTA_RE = re.compile(r"([0-9]+(\.[0-9]*)?)\s*([a-zA-Z]+)")

def parse_certificate(fileobj):
    if isinstance(fileobj, str):
        fileobj = open(fileobj, "r")
        owns = True
    else:
        owns = False
    try:
        binary_data = pem.readPemFromFile(fileobj)
        decoded = decoder.decode(binary_data,
                                 asn1Spec=x509.Certificate())[0]

        hasher = hashlib.new("sha1")
        hasher.update(binary_data)
        fingerprint = hasher.hexdigest()

        return decoded, fingerprint
    finally:
        if owns:
            fileobj.close()

def parse_date(date):
    date = str(date)
    if isinstance(date, GeneralizedTime):
        return datetime.strptime(date, "%Y%m%d%H%M%SZ")
    else:
        return datetime.strptime(date, "%y%m%d%H%M%SZ")

def extract_validity(cert):
    tbs = cert.getComponentByName("tbsCertificate")
    validity = tbs.getComponentByName("validity")

    not_before = validity.getComponentByName("notBefore").getComponent()
    not_after = validity.getComponentByName("notAfter").getComponent()

    return parse_date(not_before), parse_date(not_after)

def get_ttl(not_after):
    return not_after - datetime.utcnow()

def parse_config_timedelta(s):
    parsed = CONFIG_TIMEDELTA_RE.match(s)
    if parsed is None:
        raise ValueError("Not a valid time interval: {}".format(s))

    count, _, unit = parsed.groups()
    if unit.endswith("s") and unit != "s":
        unit = unit[:-1]
    count = float(count)

    days_factor = {
        "day": 1,
        "week": 7,
        "year": 365
    }.get(unit, 0)
    seconds_factor = {
        "minute": 60,
        "min": 60,
        "hour": 3600,
        "h": 3600,
        "s": 1,
        "second": 1
    }.get(unit, 0)

    if days_factor == 0 and seconds_factor == 0:
        raise ValueError("Unknown unit: {}".format(unit))

    return timedelta(days=count*days_factor,
                     seconds=count*seconds_factor)

def format_fingerprint(fingerprint):
    octets = int(len(fingerprint)/2)
    formatted = ""
    for octet_idx in range(octets):
        formatted += fingerprint[octet_idx*2:(octet_idx+1)*2] + ":"
    return formatted[:-1].upper()

def construct_warning_mail(responsible, warnlist, fromaddr, subjectfmt):
    mail = MIMEMultipart()
    mail["To"] = responsible
    mail["From"] = fromaddr
    mail["Date"] = format_date_time(timegm(datetime.utcnow().utctimetuple()))

    warnlist = sorted(warnlist, key=lambda x: x[-1])
    next_expiry = warnlist[0][-1]

    mail["Subject"] = subjectfmt.format(
        timedelta=str(next_expiry),
        next_expiry=str(datetime.utcnow()+next_expiry),
        warn_count=len(warnlist))

    text = """Dear administrator,

The following certificates are about to expire:
"""

    for nth, (filename, cert, fingerprint, not_after, ttl) in enumerate(warnlist):
        text += """
{nth:2d}. {filename}
    SHA1: {fingerprint}
    Remaining time: {ttl}
    Expiry at: {expiry}
""".format(
                                nth=nth+1,
                                filename=os.path.basename(filename),
                                fingerprint=
                                    format_fingerprint(fingerprint),
                                ttl=ttl,
                                expiry=not_after)

    text += """
sincerely yours,
    certwatch.py"""

    mime_text = MIMEText(text)
    mime_text.set_charset("utf-8")
    mail.attach(mime_text)

    return mail

def log_error(msg_base, exc, logging=logging):
    if not hasattr(exc, "smtp_error"):
        logging.error("%s: %s", msg_base, exc)
    else:
        try:
            logging.error("%s: %s", msg_base, exc.smtp_error.decode())
        except UnicodeDecodeError as err:
            logging.error("%s: (decode of error message failed) %s", msg_base, exc.smtp_error)

def log_smtp_error(exc, logging=logging):
    log_error("Could not connect to SMTP",
              exc,
              logging=logging)

def log_send_error(exc, logging=logging):
    log_error("While sending mail",
              exc,
              logging=logging)

if __name__ == "__main__":
    import argparse
    import configparser

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-c", "--config",
        default="/etc/certwatch.conf",
        help="Config file to use [defaults to /etc/certwatch.conf]"
    )
    parser.add_argument(
        "-v", "--verbose",
        action="count",
        default=0,
        help="Increase verbosity level"
    )

    args = parser.parse_args()

    LEVEL_MAP = {
        0: 'ERROR',
        1: 'WARNING',
        2: 'INFO',
        3: 'DEBUG'
    }

    logging.basicConfig(format='{}: [%(levelname)s] %(message)s'.format(os.path.basename(sys.argv[0])),
                        level=LEVEL_MAP.get(args.verbose, 'DEBUG'))

    args.config = open(args.config, "r")

    conf_parser = configparser.ConfigParser()

    conf_parser["DEFAULT"] = {
        "responsible": "root@localhost",
        "warn": "4 weeks"
    }

    conf_parser.read_file(args.config)

    warnings = {}

    certpath = conf_parser.get("DEFAULT", "certdir", fallback="/")
    subjectfmt = conf_parser.get(
        "DEFAULT", "subject",
        fallback="In {timedelta}, the next certificate will expire")

    for section in conf_parser.sections():
        filename = os.path.join(certpath, os.path.expanduser(section))
        logging.debug("reading: %s", filename)
        try:
            cert, fingerprint = parse_certificate(filename)
        except FileNotFoundError as err:
            logging.error(str(err))
            continue
        _, not_after = extract_validity(cert)
        ttl = get_ttl(not_after)
        logging.info("cert %s expires in %s",
                     filename, ttl)

        warn_ttl = conf_parser.get(section, "warn")
        if warn_ttl.strip() != "never":
            try:
                warn_ttl = parse_config_timedelta(warn_ttl)
            except ValueError:
                logging.error("config error: could not parse time delta: %s", warn_ttl)
                continue
            if warn_ttl >= ttl:
                responsible = conf_parser.get(section, "responsible")
                logging.debug("cert %s is below warning threshold, "
                              "notifying %s",
                              filename,
                              responsible)
                warnings.setdefault(responsible, []).append(
                    (filename, cert, fingerprint, not_after, ttl)
                )

    smtp_host = conf_parser.get("DEFAULT", "smtp_host", fallback="localhost")
    if not smtp_host.strip():
        logging.error("Invalid smtp host name: {!r}".format(smtp_host))
        sys.exit(1)
    smtp_port = conf_parser.getint("DEFAULT", "smtp_port", fallback=0)
    try:
        if conf_parser.getboolean("DEFAULT", "smtp_ssl", fallback=False):
            smtp = smtplib.SMTP_SSL(smtp_host, smtp_port)
        else:
            smtp = smtplib.SMTP(smtp_host, smtp_port)
            if conf_parser.getboolean("DEFAULT", "smtp_starttls", fallback=True):
                smtp.starttls()
        if conf_parser.has_option("DEFAULT", "smtp_helo"):
            helo = conf_parser["DEFAULT"]["smtp_helo"]
            try:
                smtp.ehlo(helo)
            except:
                smtp.helo(helo)
    except ConnectionError as err:
        log_smtp_error(err)
        sys.exit(1)
    except TimeoutError as err:
        log_smtp_error(err)
        sys.exit(1)
    except smtplib.SMTPException as err:
        log_smtp_error(err)
        sys.exit(1)
    except Exception as err:
        logging.error("while connecting to smtp server:")
        logging.exception(err)
        sys.exit(1)

    try:
        if conf_parser.has_option("DEFAULT", "smtp_user"):
            smtp.login(conf_parser["DEFAULT"]["smtp_user"], conf_parser.get("DEFAULT", "smtp_password", fallback=""))
    except smtplib.SMTPAuthenticationError as err:
        log_smtp_error(err)
        sys.exit(1)

    mail_from = conf_parser.get("DEFAULT", "from")

    for responsible, warnlist in warnings.items():
        mail = construct_warning_mail(
            responsible,
            warnlist,
            mail_from,
            subjectfmt
        )

        logging.debug("sending mail: to=<%s>, from=<%s>, \n%s",
                      responsible,
                      mail_from,
                      mail)
        try:
            smtp.sendmail(mail_from, responsible, mail.as_string())
        except smtplib.SMTPException as err:
            log_send_error(err)
            continue
        except TimeoutError as err:
            log_send_error(err)
            sys.exit(1)
        except ConnectionError as err:
            log_send_error(err)
            sys.exit(1)
        except Exception as err:
            log_send_error(err)
            sys.exit(1)

    smtp.quit()


from datetime import datetime, timedelta

# This function calculates the available time slots for scheduling a meeting
def getAvailableSlotsFromCalendar(calendar, rangeCalendar, meetingTime):
    """
    :param calendar: list of lists of strings,which represent the calendar with the meetings time
    :param rangeCalendar: list of strings, which represent the start and end time of the calendar(range)
    :param meetingTime: int, which represents the duration of the meeting
    :return: list of lists of strings, which represent the start and end time of the available slots
    """
    startTimeCalendar = datetime.strptime(rangeCalendar[0], '%H:%M')
    endTimeCalendar = datetime.strptime(rangeCalendar[1], '%H:%M')
    availableSlots = []
    # here iterate through the booked time slots in the calendar and checking the time slots between them.
    # If there is a gap that is at least as long as the meeting time,then add it to the available slots
    for i, slot in enumerate(calendar):
        if i == 0:
            if startTimeCalendar < datetime.strptime(slot[0], '%H:%M'):
                freeSlot = [startTimeCalendar.strftime('%H:%M'), slot[0]]
                if (datetime.strptime(freeSlot[1], '%H:%M') - datetime.strptime(freeSlot[0], '%H:%M')) >= timedelta(
                        minutes=meetingTime):
                    availableSlots.append(freeSlot)
        else:
            prevSlot = calendar[i - 1]
            freeSlot = [prevSlot[1], slot[0]]
            if (datetime.strptime(freeSlot[1], '%H:%M') - datetime.strptime(freeSlot[0], '%H:%M')) >= timedelta(
                    minutes=meetingTime):
                availableSlots.append(freeSlot)
        # if there is a gap between the last slot and the end of the calendar range then add it to the available slots
        if i == len(calendar) - 1:
            if endTimeCalendar > datetime.strptime(slot[1], '%H:%M'):
                freeSlot = [slot[1], endTimeCalendar.strftime('%H:%M')]
                if (datetime.strptime(freeSlot[1], '%H:%M') - datetime.strptime(freeSlot[0], '%H:%M')) >= timedelta(
                        minutes=meetingTime):
                    availableSlots.append(freeSlot)
    return availableSlots

# This function calculates the available time slots in the calendar where meetings
# could potentially be scheduled, given a specified meeting duration.
def getMeetingSlots(calendar1, calendar1Range, calendar2, calendar2Range, meetingTime):
    """
    :param calendar1:  list of lists of strings,which represent the calendar with the meetings time from the person1
    :param calendar1Range: list of strings, which represent the start and end time of the calendar(range) from the person1
    :param calendar2: list of lists of strings,which represent the calendar with the meetings time from the person2
    :param calendar2Range: list of strings, which represent the start and end time of the calendar(range) from the person2
    :param meetingTime: int, which represents the duration of the meeting
    :return: list of lists of strings, which represent the start and end time of the available slots
    """
    availableSlots1 = getAvailableSlotsFromCalendar(calendar1, calendar1Range, meetingTime)
    availableSlots2 = getAvailableSlotsFromCalendar(calendar2, calendar2Range, meetingTime)
    meetingSlots = []
    # here iterate through the available slots of each calendar,and checking if there is any overlap between them.
    # If there is and the overlap is at least as long as the meeting time,then add it to the meeting slots
    for slot1 in availableSlots1:
        for slot2 in availableSlots2:
            # here the code gives me the overlapping time range between the two time slots
            startTime = max(datetime.strptime(slot1[0], '%H:%M'), datetime.strptime(slot2[0], '%H:%M'))
            endTime = min(datetime.strptime(slot1[1], '%H:%M'), datetime.strptime(slot2[1], '%H:%M'))
            if (endTime - startTime) >= timedelta(minutes=meetingTime):
                meetingSlots.append([startTime.strftime('%H:%M'), endTime.strftime('%H:%M')])
    return meetingSlots

calendar1 = [['9:00', '10:30'], ['12:00', '13:00'], ['16:00', '18:00'], ['19:00', '20:00']]
calendar1Range = ['9:00', '20:00']
calendar2 = [['10:00', '11:30'], ['12:30', '14:30'], ['14:30', '15:00'], ['16:00', '17:00']]
calendar2Range = ['10:00', '18:30']
meetingTime = 30

meetingSlots = getMeetingSlots(calendar1, calendar1Range, calendar2, calendar2Range, meetingTime)
print(meetingSlots)

# datetime.strptime: It's a method that creates a datetime object from a string representing a date and time, using a specified format.
# For example, datetime.strptime('9:00', '%H:%M') will create a datetime object with hour = 9 and minute = 0.

# timedelta: It's a class that represents a duration, or the difference between two dates or times. It's used for performing arithmetic with dates and times.
# For example, timedelta(minutes=30) creates a duration of 30 minutes.

# startTimeCalendar.strftime('%H:%M'): It's a method that creates a string representation of a datetime object, using a specified format.
# For example, startTimeCalendar.strftime('%H:%M') will create a string in the format of "hour:minute" (e.g. "09:00").


import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import statsmodels.api as sm
from sklearn.model_selection import train_test_split

tabla =  pd.read_csv("data.csv", low_memory=False) #Created data frame from our dataset
tabla = tabla[["year","transmission_type", "fuel_type", "co_emissions", "co2","engine_capacity","noise_level"]] #Filters dataframe by given columns
tabla.dropna(inplace=True) #Deletes rows when it is empty

tabla["transmission_type"].replace({"Manual": 0, "Automatic": 1}, inplace=True)
tabla["fuel_type"].replace({"Diesel": 0, "Petrol": 1, "LPG": 2, "CNG": 3, "Petrol Hybrid": 4, "Diesel Electric": 5, "Electricity": 6, "Electricity/Diesel":7,"Electricity/Petrol":8, "LPG / Petrol": 9, "Petrol / E85":10, "Petrol / E85 (Flex Fuel)":11, "Petrol Electric": 12}, inplace=True)

train, test = train_test_split(tabla, test_size=0.2) #Split our data in 80 / 20 ratio for training and testing of our model

x = train[["year","transmission_type","engine_capacity","noise_level"]]

y = train["co2"]
x = sm.add_constant(x)
modelo = sm.OLS(y, x).fit()

predictions = modelo.predict(x)  # Con el modelo, se calculan las Y´s
print("==============================================================================")
print(modelo.summary())

tabla2 = test.assign(calculated_co2 = lambda x: 8849.3913 -4.4589 * x.year + 11.2728 * x.transmission_type + 0.0469 * x.engine_capacity + 2.5492 * x.noise_level)
tabla2 = tabla2[["year","transmission_type","engine_capacity","noise_level", "co2", "calculated_co2"]]
tabla2 = tabla2.assign(error = lambda x: abs(((x.co2 - x.calculated_co2)/x.co2)))
print("==============================================================================")
print(tabla2.head())
print("==============================================================================")
print("El error promedio es: ", np.mean(tabla2["error"]))

tabla2 = tabla2.head(20)
y1 = tabla2["co2"]
y2 = tabla2["calculated_co2"]
independiente = np.arange(len(y1))

plt.plot(independiente,y1, c="b", label = "Real")
plt.plot(independiente,y2, c="r", label = "Calculado")
plt.xlabel("Valores en X")
plt.ylabel("Valores en Y")
plt.title("Valor real vs Calculado")
plt.legend(loc='upper left');
plt.show()


import copy
import numpy as np
from numpy.core.defchararray import mod
import pandas as pd
import matplotlib.pyplot as plt

import models

class BaseTrackFactory():
    """ Track Factory Base Class """
    def __init__(self, track, **kwargs):
        self.track = track
        self.param = kwargs

    def set_attr(self, **kwargs):
        if "model_factory" in kwargs:
            self.model_factory = kwargs["model_factory"]
        if "tracker" in kwargs:
            self.tracker = kwargs["tracker"]

    def create(self, obs):

        if hasattr(self, "tracker"):
            timestamp = self.tracker.timestamp
        else:
            timestamp = 0

        return self.track(
            obs,
            self.model_factory.create(obs),
            timestamp=timestamp,
            **self.param
        )

class BaseTrack(models.BaseExporter):
    """ Base Track

        ref) Design and Analysis of Modern Tracking Systems
                    6.2 Track Score Function
                    6.3 Gating
                    6.4 Global Nearest Neighbor Method

     * Use Kalman Filter
     * Use Log Likelihood Ratio as Score
    """
    trk_id_counter = 0
    @classmethod
    def _generate_id(cls):
        BaseTrack.trk_id_counter+=1
        return BaseTrack.trk_id_counter

    @classmethod
    def initialize(cls):
        BaseTrack.trk_id_counter = 0

    def __init__(self, obs, model, **kwargs):
        # set param
        if "gate" not in kwargs:
            kwargs["gate"] = None

        if "timestamp" not in kwargs:
            kwargs["timestamp"] = 0

        self.param = kwargs
        self.sensor = obs.sensor
        # set data
        self.obs_list = [copy.deepcopy(obs)]
        self.scr_list = [self._calc_init_score(obs)]
        self.cnt_list = [self.param["timestamp"]]
        self.model = model
        self.mdl_list = [copy.deepcopy(self.model)]
        self.trk_id = self._generate_id()

    def get_id(self):
        return self.trk_id

    def to_record(self, timestamp, scan_id):
        series = super().to_record(timestamp, scan_id)
        # id, score
        trk_id = self.get_id()
        obs_id = self.obs_list[-1].get_id() if self.obs_list[-1] else -1
        value=[trk_id, obs_id, self.scr_list[-1]]
        label=["TRK_ID", "OBS_ID", "SCORE"]
        series = series.append( pd.Series(value, index=label) )
        return series.append( self.model.to_record() )

    @staticmethod
    def from_record_and_factory(series, track_factory, model_factory):
        assert isinstance(series, pd.Series), "series is invalid, actual:"  + str(type(series))
        model = models.KalmanModel.from_record_and_factory(series, model_factory)
        class DummyTrack():
            def __init__(self, model, trk_id):
                self.model = model
                self.trk_id = trk_id
            def get_id(self):
                return self.trk_id
        return DummyTrack(model=model, trk_id=series["TRK_ID"])

    def assign(self, obs):
        # set data
        self.obs_list.append(copy.deepcopy(obs))
        self.mdl_list.append(copy.deepcopy(self.model))
        self.cnt_list.append(self.cnt_list[-1]+1)
        self.scr_list.append(self._calc_match_score(obs))
        # update model
        self.model.update(obs)

    def unassign(self, sensor):
        # set data
        self.obs_list.append(None)
        self.mdl_list.append(copy.deepcopy(self.model))
        self.cnt_list.append(self.cnt_list[-1]+1)
        if sensor is None or not sensor.is_trk_in_range(self):
            self.scr_list.append(self._calc_not_in_range_score())
        else:
            self.scr_list.append(self._calc_miss_score())
        # update model
        self.model.update(None)

    def get_gate(self, obs):
        dist, detS, M = self.model.norm_of_residual(obs)
        gate = obs.sensor.calc_ellipsoidal_gate(detS, M)

        if self.param["gate"]:
            gate = self.param["gate"]
        return (gate, dist)

    def is_in_gate(self, obs):
        gate, dist = self.get_gate(obs)
        return gate > dist

    def calc_match_price(self, obs):
        dist, detS, M = self.model.norm_of_residual(obs)
        gate = obs.sensor.calc_ellipsoidal_gate(detS, M)
        if False:
            # p.339 version (using term for penalizing tracks with greater prediction uncertainty)
            # however, in some case, penalty term is too strong for associate new track with next obs
            # because new track has greater prediction uncertainty P0, esspecially velocity uncertainty.
            return gate - dist - np.log(detS)
        else:
            # p.340 version (simple)
            return gate - dist

    def judge_confirmation(self):
        raise NotImplementedError

    def judge_deletion(self):
        raise NotImplementedError

    def _calc_init_score(self, obs):
        return obs.sensor.calc_LLR0()

    def _calc_match_score(self, obs):
        log_gij = self.model.gaussian_log_likelihood(obs)
        return self.scr_list[-1] + obs.sensor.calc_match_dLLR(log_gij)

    def _calc_miss_score(self):
        return self.scr_list[-1] + self.sensor.calc_miss_dLLR()

    def _calc_not_in_range_score(self):
        return self.scr_list[-1]

    def plot_obs_list(self):
        plt.plot(
            [obs.y[0] if obs is not None else None for obs in self.obs_list ],
            [obs.y[1] if obs is not None else None for obs in self.obs_list ],
            marker="D", color="r", alpha=.5, linestyle="None"
        )

    def plot_scr_list(self):
        plt.plot(
            self.cnt_list,
            self.scr_list,
            marker="D", color="r", alpha=.5, linestyle="None"
        )

    def plot_mdl_list(self):
        plt.plot(
            [mdl.x[0] if mdl is not None else None for mdl in self.mdl_list ],
            [mdl.x[1] if mdl is not None else None for mdl in self.mdl_list ],
            marker="D", color="g", alpha=.5, linestyle="None"
        )

class SimpleManagedTrack(BaseTrack):
    """ Simple Managed Track

        ref) Design and Analysis of Modern Tracking Systems
                    6.2.4 Score-Based Track Confirmation and Deletion p.334
    """
    def __init__(self, obs, model_factory, **kwargs):
        super().__init__(obs, model_factory, **kwargs)

        # parameter for another deletion argorithm of p.334
        if "ND" not in self.param:
            self.param["ND"] = 5    # continuous miss number for delete

    def judge_confirmation(self):
        return True

    def judge_deletion(self):
        # another deletion argorithm of p.334
        ND = self.param["ND"]
        return self.obs_list[-ND:] == [None]*ND

class ScoreManagedTrack(BaseTrack):
    """ LLR Score Managed Track

        ref) Design and Analysis of Modern Tracking Systems
                    6.2.4 Score-Based Track Confirmation and Deletion p.333
    """
    def __init__(self, obs, model_factory, **kwargs):
        super().__init__(obs, model_factory, **kwargs)

        # parameter for confirmation and deletion argorithm of p.333
        if "PFD" not in self.param:
            self.param["PFD"] = 1.e-3
        if "alpha" not in self.param:
            self.param["alpha"] = 4/3600/1000
        if "beta" not in self.param:
            self.param["beta"] = 0.1

        self.param["THD"] = np.log( self.param["PFD"] )
        self.param["T1"] = np.log( self.param["beta"] / (1-self.param["alpha"]) )
        self.param["T2"] = np.log( (1-self.param["beta"]) / self.param["alpha"] )

    def judge_confirmation(self):
        # use score for confirmation
        return self.scr_list[-1] > self.param["T2"]

    def judge_deletion(self):
        # use score for deletion
        is_del_a = self.scr_list[-1] < self.param["T1"]
        is_del_b =  self.scr_list[-1] - max(self.scr_list) < self.param["THD"]
        return is_del_a or is_del_b

class PDATrack(BaseTrack):
    """ LLR Track for PDA

    ref) Design and Analysis of Modern Tracking Systems
                    6.6.1 The PDA Method
                    6.6.2 Extension to JPDA
                    6.6.4 PDA Track Initiation and Deletion
    """
    def __init__(self, obs, model_factory, **kwargs):
        super().__init__(obs, model_factory, **kwargs)
        self.pt_list = [0.5]

        # set param
        if "P22" not in self.param:
            self.param["P22"] = 0.98

    def assign(self, obs_dict):
        # track score
        score = 0.0
        for obs, ratio in obs_dict.items():
            if obs:
                score += ratio * (self._calc_match_score(obs) - self.scr_list[-1])
            else:
                score += ratio * (self._calc_miss_score() - self.scr_list[-1])

        # comfirmation and deletion parameter
        delta = 0.0
        count = 0
        for obs, ratio in obs_dict.items():
            if obs:
                count += 1
                gate, _ = self.get_gate(obs)
                vg = self.model.volume_of_ellipsoidal_gate( obs, gate )
                delta -= vg * np.exp( self.model.gaussian_log_likelihood(obs) )
            else:
                pass
        if count:
            delta /= count - self.sensor.param["PD"] * self.pt_list[-1]
            delta += 1.0
            delta *= self.sensor.param["PD"]
        else:
            delta = self.sensor.param["PD"]
        ptkk = (1-delta)/(1-delta*self.pt_list[-1])*self.pt_list[-1]
        p22 = self.param["P22"]

        #  # set data
        self.obs_list.append(obs_dict)
        self.cnt_list.append(self.cnt_list[-1]+1)
        self.scr_list.append(score)
        self.pt_list.append( p22*ptkk + (1-p22)*(1-ptkk) )

        # # update model
        self.model.update(obs_dict)

    def judge_confirmation(self):
        # use pt for confirmation
        return self.pt_list[-1] > 0.95

    def judge_deletion(self):
        # use pt for deletion
        return self.pt_list[-1] < 0.4

class FormationTrack(SimpleManagedTrack):
    """ Formation Track for Formation Group Tracking

        ref) Radar Data Processing with Applications
                10.5 Formation Group Tracking

     * Use Kalman Filter
     * Use Log Likelihood Ratio as Score
    """
    def __init__(self, obs, model_factory, **kwargs):
        super().__init__(obs, model_factory, **kwargs)
        self.group_model=None

    def __sub__(self, other):
        # dx, _ = models.ModelType.diff_x(self.model.x, self.model._x_type, other.model.x, other.model._x_type)
        dx = self.model.x - other.model.x
        P  = self.model.P + other.model.P
        return dx @ np.linalg.inv(P) @ dx

    def calc_match_price(self, obs):
        if self.is_group():
            dist, detS, M = self.group_model.norm_of_residual(obs)
            gate = obs.sensor.calc_ellipsoidal_gate(detS, M)
            return gate - dist
        else:
            return super().calc_match_price(obs)

    def get_gate(self, obs):
        if self.is_group():
            dist, detS, M = self.group_model.norm_of_residual(obs)
            gate = obs.sensor.calc_ellipsoidal_gate(detS, M)
            if self.param["gate"]:
                gate = self.param["gate"]
        else:
            gate, dist = super().get_gate(obs)
        return (gate, dist)

    def preprocess_for_grouping(self):
        self.group_model=None

    def set_group_model(self, model):
        for i, obs in enumerate(reversed(self.obs_list)):
            if obs is not None:
                x = obs.y @ model.H
                model.x[model.H.sum(axis=0)>0] = x[model.H.sum(axis=0)>0]
                [ model._predict_step() for _ in range(i+1) ]
                self.group_model = model
                break

    def is_group(self):
        return self.group_model is not None

class MultiSensorScoreManagedTrack(BaseTrack):
    """ Multi Sensor LLR Track

        ref) Design and Analysis of Modern Tracking Systems
                    9.5 General Expression for Multisensor Data Association

     * Use Kalman Filter
     * Use Log Likelihood Ratio as Score
     * Observation to Track Association Based System
    """
    def calc_match_price(self, obs):
        raise NotImplementedError

class TrackEvaluator():
    """ Evaluate Each Track

    * Evaluate Gating, Conformation, Deletion etc.
    """

    def __init__(self, sensor, model_factory, track_factory, target, R):
        """
        Arguments:
        sensor {Sensor} -- sensor
        model_factory {ModelFactory} -- model factory
        track_factory {TrackFactory} -- track factory
        target {Target} -- target
        R {np.array} -- observation error covariance matrix
        """
        self.track_factory = track_factory
        self.track_factory.set_attr(model_factory=model_factory)
        self.target = target
        self.R = R
        self.sensor = sensor

    def _dT(self):
        return self.track_factory.model_factory.dT

    def _initialize_simulation(self):
        self.tgt_list = []
        self.obs_list = []
        self.trk_list = []
        tgt = self.target
        obs = models.Obs(np.random.multivariate_normal(tgt.x[:len(self.R)], self.R), self.R, self.sensor)
        trk = self.track_factory.create( obs )
        assert tgt.x.shape == trk.model.x.shape, "target.x.shape and model.x.shape invalid, actual:" + str(tgt.x.shape) + str(trk.model.x.shape)
        return (tgt, obs, trk)

    def _update(self, tgt, obs, trk):
        # update target
        tgt.update(self._dT())

        # update observation
        obs = models.Obs(np.random.multivariate_normal(tgt.x[:len(self.R)], self.R), self.R, self.sensor)

        dist, detS, M = trk.model.norm_of_residual(obs)
        gate = obs.sensor.calc_ellipsoidal_gate(detS, M)
        mch_scr = trk._calc_match_score(obs)
        ini_scr = trk._calc_init_score(obs)

        # track update
        trk.assign(obs)
        return (tgt, obs, trk, dict(dist=dist, gate=gate, mch_scr=mch_scr, ini_scr=ini_scr))

    def plot_score(self, n_count=10):
        # init
        tgt, obs, trk = self._initialize_simulation()
        data_tbl = []

        # simulate
        i_count = n_count
        while i_count>0:

            tgt, obs, trk, data = self._update(tgt, obs, trk)
            data_tbl.append( data )
            i_count -= 1

        dist_list = [ data.get("dist") for data in data_tbl ]
        gate_list = [ data.get("gate") for data in data_tbl ]
        mch_scr_list = [ data.get("mch_scr") for data in data_tbl ]
        ini_scr_list = [ data.get("ini_scr") for data in data_tbl ]

        _, (axU, axD) = plt.subplots(nrows=2, sharex=True)

        axU.plot(dist_list, marker="D", label="dist")
        axU.plot(gate_list, marker="D", label="gate")
        axU.legend()

        axD.plot(mch_scr_list, marker="D", label="match_score")
        axD.plot(ini_scr_list, marker="D", label="init_score")
        axD.legend()

        plt.show()

    # def plot_stat_score(self, n_count=10, n_run=10):
    #     data_tbl_runs = []
    #     i_run = n_run
    #     while i_run>0:

    #         # init
    #         tgt, obs, trk = self._initialize_simulation()
    #         data_tbl = []

    #         # simulate
    #         i_count = n_count
    #         while i_count>0:

    #             tgt, obs, trk, data = self._update(tgt, obs, trk)
    #             data_tbl.append( data )
    #             i_count -= 1

    #         data_tbl_runs.append( data_tbl )
    #         i_run -= 1

    #     # calc average
    #     dist_list=[]
    #     gate_list=[]
    #     mch_scr_list=[]
    #     ini_scr_list=[]
    #     for ii in range(n_count):
    #         dist_list.append( sum([data_tbl[ii].get("dist") for data_tbl in data_tbl_runs])/n_run )
    #         gate_list.append( sum([data_tbl[ii].get("gate") for data_tbl in data_tbl_runs])/n_run )
    #         mch_scr_list.append( sum([data_tbl[ii].get("mch_scr") for data_tbl in data_tbl_runs])/n_run )
    #         ini_scr_list.append( sum([data_tbl[ii].get("ini_scr") for data_tbl in data_tbl_runs])/n_run )

    #     _, (axU, axD) = plt.subplots(nrows=2, sharex=True)

    #     axU.plot(dist_list, marker="D", label="dist")
    #     axU.plot(gate_list, marker="D", label="gate")
    #     axU.legend()

    #     axD.plot(mch_scr_list, marker="D", label="match_score")
    #     axD.plot(ini_scr_list, marker="D", label="init_score")
    #     axD.legend()

    #     plt.show()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""HLS color space.
"""

from __future__ import annotations

import math

import torch
from torch import Tensor

from torchkit.core.factory import TRANSFORMS

__all__ = [
    "hls_to_rgb",
    "HlsToRgb",
    "rgb_to_hls",
    "RgbToHls"
]

# MARK: - HlsToRgb

def hls_to_rgb(image: Tensor) -> Tensor:
    """Convert a HLS image to RGB. Image data is assumed to be in the range
    of [0.0, 1.0].

    Args:
        image (Tensor):
            HLS image to be converted to RGB with shape [*, 3, H, W].

    Returns:
        (Tensor):
            RGB version of the image with shape [*, 3, H, W].

    Example:
        >>> input  = torch.rand(2, 3, 4, 5)
        >>> output = hls_to_rgb(input)  # [2, 3, 4, 5]
    """
    if not isinstance(image, Tensor):
        raise TypeError(f"Input type is not a Tensor. Got: {type(image)}")

    if len(image.shape) < 3 or image.shape[-3] != 3:
        raise ValueError(f"Input size must have a shape of [*, 3, H, W]. "
                         f"Got: {image.shape}")

    if not torch.jit.is_scripting():
        # weird way to use globals compiling with JIT even in the code not used by JIT...
        # __setattr__ can be removed if pytorch version is > 1.6.0 and then use:
        # hls_to_rgb.HLS2RGB = hls_to_rgb.HLS2RGB.to(image.device)
        hls_to_rgb.__setattr__("HLS2RGB", hls_to_rgb.HLS2RGB.to(image))  # type: ignore
        _HLS2RGB = hls_to_rgb.HLS2RGB  # type: ignore
    else:
        _HLS2RGB = torch.tensor([[[0.0]], [[8.0]], [[4.0]]], device=image.device, dtype=image.dtype)  # [3, 1, 1]

    im = image.unsqueeze(-4)
    h  = torch.select(im, -3, 0)
    l  = torch.select(im, -3, 1)
    s  = torch.select(im, -3, 2)
    h *= 6 / math.pi  # h * 360 / (2 * math.pi) / 30
    a  = s * torch.min(l, 1.0 - l)

    # kr = (0 + h) % 12
    # kg = (8 + h) % 12
    # kb = (4 + h) % 12
    k = (h + _HLS2RGB) % 12

    # l - a * max(min(min(k - 3.0, 9.0 - k), 1), -1)
    mink = torch.min(k - 3.0, 9.0 - k)
    return torch.addcmul(l, a, mink.clamp_(min=-1.0, max=1.0), value=-1)

# noinspection PyMethodMayBeStatic
@TRANSFORMS.register(name="hls_to_rgb")
class HlsToRgb(torch.nn.Module):
    """Convert an image from HLS to RGB. Image data is assumed to be in the
    range of [0.0, 1.0].

    Shape:
        - input:  [*, 3, H, W]
        - output: [*, 3, H, W]

    Reference:
        https://en.wikipedia.org/wiki/HSL_and_HSV

    Examples:
        >>> input  = torch.rand(2, 3, 4, 5)
        >>> rgb    = HlsToRgb()
        >>> output = rgb(input)  # [2, 3, 4, 5]
    """

    def forward(self, image: Tensor) -> Tensor:
        return hls_to_rgb(image)

# MARK: - RgbToHls

def rgb_to_hls(image: Tensor, eps: float = 1e-8) -> Tensor:
    """Convert a RGB image to HLS. Image data is assumed to be in the range
    of [0.0, 1.0].

    NOTE: this method cannot be compiled with JIT in pytohrch < 1.7.0

    Args:
        image (Tensor):
            RGB image to be converted to HLS with shape [*, 3, H, W].
        eps (float):
            Epsilon value to avoid div by zero.

    Returns:
        hls (Tensor):
            HLS version of the image with shape [*, 3, H, W].

    Example:
        >>> input  = torch.rand(2, 3, 4, 5)
        >>> output = rgb_to_hls(input)  # 2x3x4x5
    """
    if not isinstance(image, Tensor):
        raise TypeError(f"Input type is not a Tensor. Got: {type(image)}")

    if len(image.shape) < 3 or image.shape[-3] != 3:
        raise ValueError(f"Input size must have a shape of [*, 3, H, W]. "
                         f"Got: {image.shape}")

    if not torch.jit.is_scripting():
        # weird way to use globals compiling with JIT even in the code not used by JIT...
        # __setattr__ can be removed if pytorch version is > 1.6.0 and then use:
        # rgb_to_hls.RGB2HSL_IDX = hls_to_rgb.RGB2HSL_IDX.to(image.device)
        rgb_to_hls.__setattr__("RGB2HSL_IDX", rgb_to_hls.RGB2HSL_IDX.to(image))  # type: ignore
        _RGB2HSL_IDX = rgb_to_hls.RGB2HSL_IDX  # type: ignore
    else:
        _RGB2HSL_IDX = torch.tensor([[[0.0]], [[1.0]], [[2.0]]], device=image.device, dtype=image.dtype)  # [3, 1, 1]

    # maxc: Tensor  # not supported by JIT
    # imax: Tensor  # not supported by JIT
    maxc, imax = image.max(-3)
    minc       = image.min(-3)[0]

    # h: Tensor  # not supported by JIT
    # l: Tensor  # not supported by JIT
    # s: Tensor  # not supported by JIT
    # image_hls: Tensor  # not supported by JIT
    if image.requires_grad:
        l_ = maxc + minc
        s  = maxc - minc
        # weird behaviour with undefined vars in JIT...
        # scripting requires image_hls be defined even if it is not used :S
        h  = l_  # assign to any image...
        image_hls = l_  # assign to any image...
    else:
        # define the resulting image to avoid the torch.stack([h, l, s])
        # so, h, l and s require inplace operations
        # NOTE: stack() increases in a 10% the cost in colab
        image_hls = torch.empty_like(image)
        h         = torch.select(image_hls, -3, 0)
        l_        = torch.select(image_hls, -3, 1)
        s         = torch.select(image_hls, -3, 2)
        torch.add(maxc, minc, out=l_)  # l = max + min
        torch.sub(maxc, minc, out=s)  # s = max - min

    # precompute image / (max - min)
    im = image / (s + eps).unsqueeze(-3)

    # epsilon cannot be inside the torch.where to avoid precision issues
    s  /= torch.where(l_ < 1.0, l_, 2.0 - l_) + eps  # saturation
    l_ /= 2  # luminance

    # note that r,g and b were previously div by (max - min)
    r = torch.select(im, -3, 0)
    g = torch.select(im, -3, 1)
    b = torch.select(im, -3, 2)
    # h[imax == 0] = (((g - b) / (max - min)) % 6)[imax == 0]
    # h[imax == 1] = (((b - r) / (max - min)) + 2)[imax == 1]
    # h[imax == 2] = (((r - g) / (max - min)) + 4)[imax == 2]
    cond = imax.unsqueeze(-3) == _RGB2HSL_IDX
    if image.requires_grad:
        h = torch.mul((g - b) % 6, torch.select(cond, -3, 0))
    else:
        torch.mul((g - b).remainder(6), torch.select(cond, -3, 0), out=h)
    h += torch.add(b - r, 2) * torch.select(cond, -3, 1)
    h += torch.add(r - g, 4) * torch.select(cond, -3, 2)
    # h = 2.0 * math.pi * (60.0 * h) / 360.0
    h *= math.pi / 3.0  # hue [0, 2*pi]

    if image.requires_grad:
        return torch.stack([h, l_, s], dim=-3)
    return image_hls

# noinspection PyMethodMayBeStatic
@TRANSFORMS.register(name="rgb_to_hls")
class RgbToHls(torch.nn.Module):
    """Convert an image from RGB to HLS. Image data is assumed to be in the
    range of [0.0, 1.0].

    Shape:
        - image:  [*, 3, H, W]
        - output: [*, 3, H, W]

    Examples:
        >>> input  = torch.rand(2, 3, 4, 5)
        >>> hls    = RgbToHls()
        >>> output = hls(input)  # [2, 3, 4, 5]
    """

    def forward(self, image: Tensor) -> Tensor:
        return rgb_to_hls(image)

# Tricks to speed up a little bit the conversions by presetting small tensors
# (in the functions they are moved to the proper device)
hls_to_rgb.__setattr__("HLS2RGB",     torch.tensor([[[0.0]], [[8.0]], [[4.0]]]))  # [3, 1, 1]
rgb_to_hls.__setattr__("RGB2HSL_IDX", torch.tensor([[[0.0]], [[1.0]], [[2.0]]]))  # [3, 1, 1]


import requests

from setting import test, log
# proxies = { 'https': "http//localhost:1080"}
proxies = {"https": "http://localhost:1080", }
for index, item in enumerate(test['img_herder_merge'].find()):
    # blogger_id = item.get('url').split('com/')[-1]
    try:
        blogger_id = item.get('blogger_id')
        url = item.get('header_url')
        id = item.get('id')
        # if int(id) < 5976:
        #     continue
        # print(id, url)
        log(id, url)
        if not url:
            test['save'].insert({'id': id, 'header_url': ''})
            continue
        resp = requests.get(url, proxies=proxies)
        with open('img_header_6000/{}.png'.format(blogger_id), 'wb') as f:
            f.write(resp.content)
        test['save_img'].insert({'id':id, 'header_url': url})
        # print('save {}'.format(id))
        log('{} save {}'.format(index, id))
    except Exception as e:
        log(e)
        log('=============')
        log(id, blogger_id)


# -*- coding: utf-8 -*-
import flask
import telebot
import conf

WEBHOOK_URL_BASE = "https://{}:{}".format(conf.WEBHOOK_HOST, conf.WEBHOOK_PORT)
WEBHOOK_URL_PATH = "/{}/".format(conf.TOKEN)

bot = telebot.TeleBot(conf.TOKEN, threaded=False)  # бесплатный аккаунт pythonanywhere запрещает работу с несколькими тредами

# удаляем предыдущие вебхуки, если они были
bot.remove_webhook()

# ставим новый вебхук = Слышь, если кто мне напишет, стукни сюда — url
bot.set_webhook(url=WEBHOOK_URL_BASE+WEBHOOK_URL_PATH)

app = flask.Flask(__name__)
import telebot
from telebot import types
import threading
import sqlite3
from bs4 import BeautifulSoup
import re
import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
from sklearn.metrics.pairwise import cosine_similarity
import requests
from pprint import pprint
import time
from datetime import datetime

notes = {}

bot = telebot.TeleBot('5627814001:AAFfUSr_HramrdpQ8HCBirmYSjJ0dvaNi_g')

from fake_useragent import UserAgent
session = requests.session()
ua = UserAgent(verify_ssl=False)
headers = {'User-Agent': ua.random}
import gensim
import logging
import urllib.request
import warnings
warnings.filterwarnings('ignore')
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import TfidfVectorizer
from nltk.tokenize import wordpunct_tokenize
from pymorphy2 import MorphAnalyzer
morph = MorphAnalyzer()
import nltk
nltk.download('stopwords')
stops = stopwords.words("russian")
tfidf = TfidfVectorizer(
    analyzer="word",
    stop_words=stops, ngram_range=(1,3),min_df=3
)

#БЛОК ПОСЛЕДНИЕ НОВОСТИ. открываем сайт
print('Последние новости: ')
page_number = 1
url = f'https://vestinn.ru/news/?PAGEN_1={page_number}'
req = session.get(url, headers={'User-Agent': ua.random})
page = req.text

soup = BeautifulSoup(page, 'html.parser')

#все новости с общей страницы
news = soup.find_all('div', {'class': 'news'})
#title для последних трех новостей + ссылка

title_obj = news[0].find('a')
title = title_obj.find('div',{'class':"ellipsis__text"}).text
attrs = title_obj.attrs
href = title_obj.attrs['href']
url= 'https://vestinn.ru' + href
print(f'{title}. Подробнее по ссылке: {url}')

title_obj1 = news[1].find('a')
title1 = title_obj1.find('div',{'class':"ellipsis__text"}).text
attrs1 = title_obj1.attrs
href1 = title_obj1.attrs['href']
url1= 'https://vestinn.ru' + href1
print(f'{title1}. Подробнее по ссылке: {url1}')

title_obj2 = news[2].find('a')
title2 = title_obj2.find('div',{'class':"ellipsis__text"}).text
attrs2 = title_obj2.attrs
href2 = title_obj2.attrs['href']
url2 = 'https://vestinn.ru' + href2
print(f'{title2}. Подробнее по ссылке: {url2}')

#ПОЛНАЯ ПРОГА ДЛЯ КРАУЛЕРА.
import schedule
import datetime
import pytz
import time
import requests
import pandas as pd
import requests
from pprint import pprint
from fake_useragent import UserAgent
session = requests.session()
import time
from datetime import datetime
ua = UserAgent(verify_ssl=False)
headers = {'User-Agent': ua.random}

import sqlite3
from bs4 import BeautifulSoup
import re
def crawler():
    conn = sqlite3.connect('news.db')
    cur = conn.cursor()
    cur.execute("""
    CREATE TABLE IF NOT EXISTS texts
    (id INTEGER PRIMARY KEY AUTOINCREMENT, news_id int, pub_year int, pub_month int,
    pub_day int, title text, short_text text, full_text text, link text)
    """)
    #тэги
    cur.execute("""
    CREATE TABLE IF NOT EXISTS tags_names
    (id int PRIMARY KEY, news_id int, tags text)
    """)
    #просмотры
    cur.execute("""
    CREATE TABLE IF NOT EXISTS views
    (id int PRIMARY KEY, news_id int, views_news int)
    """)

    conn.commit()
    conn.close()
    #страница со списком новостей
    def parse_news_page_block(one_block):
        block = {}
        a = one_block.find('a')
        block['title'] = str(a.text)
        block['href'] = a.attrs['href']
        block['views_news'] = int(one_block.find('div', {'class':"news__views"}).text)
        block['pub_day'] = int(one_block.find('div', {'class':"news__date"}).text[:2])
        block['pub_month'] = int(one_block.find('div', {'class':"news__date"}).text[3:5])
        block['pub_year'] = int(one_block.find('div', {'class':"news__date"}).text[6:8])
        return block
    #страница одной новости
    def parse_one_article(block):
        url_one = 'https://vestinn.ru' + block['href']
        block['link'] = 'https://vestinn.ru' + block['href']
        block['page_id'] = int(block['href'].split('/')[-2])
        req = session.get(url_one, headers={'User-Agent': ua.random})
        page = req.text
        soup = BeautifulSoup(page, 'html.parser')
        block['full_text'] = str(soup.find('div', {'class':"readMore__content ellipsis__text"}).text)
        t = soup.find('div',{'class':"tags_list"})
        block['tags'] = str(t.find('a').text)
        short = soup.find('meta', {'property':'og:description'}).get('content')
        cleantext = re.compile('<.*?>|&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});')
        block['short_text'] = str(re.sub(cleantext, '', short))
        block['date'] = str(soup.find('meta',{'itemprop':"datePublished"}).get('datetime'))
        return block

    #Обработать N-ую страницу новостей
    def get_nth_page(page_number):
        # скачиваем
        url = f'https://vestinn.ru/news/?PAGEN_1={page_number}'

        req = session.get(url, headers={'User-Agent': ua.random})
        page = req.text
        soup = BeautifulSoup(page, 'html.parser')

        # находим новости
        news = soup.find_all('div', {'class': 'news'})

        # идем по новостям и обрабатываем их
        blocks = []
        for n in news:
            try:
                blocks.append(parse_news_page_block(n))
            except Exception as e:
                print(e)

        # идем по отдельным статьям и достаем информацию
        result = []
        for b in blocks:
            if b['href'] not in seen_news:
                    try:
                        res = parse_one_article(b)
                        res['news_id'] = b['page_id']
                        result.append(res)
                    except Exception as e:
                        print(e)
            else:
                print('Seen', b['href'])

        # возвращаем найденную информацию
        return result
    #Пишем в базу.
    def write_to_db(block):
          # сохраняем информацию по текстам
        cur.execute(
            """
            INSERT INTO texts
                (news_id, title, pub_year, pub_month, pub_day, short_text, full_text, link)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
            """, (
                block['news_id'], block['title'], block['pub_year'], block['pub_month'],
                block['pub_day'], block['short_text'],
                block['full_text'], block['link'])
        )
        conn.commit()
    #тэги
        cur.execute(
            """
            INSERT INTO tags_names
                (news_id, tags)
                VALUES (?, ?)
            """, (
                block['news_id'], block['tags'])
        )
        conn.commit()
    #просмотры
        cur.execute(
            """
            INSERT INTO views
                (news_id, views_news)
                VALUES (?,?)
            """, (
                block['news_id'], block['views_news'])
        )
        conn.commit()
        # добавляем, что такой id уже видели
        seen_news.add(block['news_id'])

    from tqdm.auto import tqdm
    conn = sqlite3.connect('news.db')
    cur = conn.cursor()

    cur.execute('SELECT news_id FROM texts')
    seen_news = set(i[0] for i in cur.fetchall())

    def run_all(n_pages):
        for i in tqdm(range(n_pages)):
            blocks = get_nth_page(i+1)
            for block in blocks:
                write_to_db(block)

    run_all(30)

def main(): #Таймер
    # Выставляем время, в которое будет запускаться краулер
    schedule.every().day.at("02:28").do(crawler)
    while True:
        schedule.run_pending()
        time.sleep(1)

if __name__ == 'main':
    # Запускаем функцию main() в отдельном потоке
    t = threading.Thread(target=main)
    t.start()
    t.join() # Ожидаем завершения потока
    # Код после выполнения функции main() продолжит выполняться

con = sqlite3.connect('news.db')  # подключение
cur = con.cursor()

id_title_text = """
SELECT news_id, title, full_text, pub_day, link  from texts
"""
cur.execute(id_title_text)

df = pd.read_sql_query(id_title_text, con=conn)

#БЛОК СТАТИСТИКИ

# #ГРАФИК САМЫЕ НАСЫЩЕННЫЕ ДНИ
current_month = time.strftime("%m")
fullest_day ="""
SELECT count(pub_day) as cnt, pub_day, pub_month
    FROM texts
        WHERE pub_month = ?
        GROUP BY pub_day
        ORDER BY cnt DESC
        LIMIT 30;
"""
cur.execute(fullest_day, (current_month, ))
cur.fetchall()

df = pd.read_sql_query(fullest_day, params = [current_month], con=con)
df.plot(x = "pub_day", y = "cnt", color='mediumseagreen', marker='o', label='День публикации')
plt.locator_params (axis='x')
plt.locator_params (axis='y')
plt.title('Соотношение количества новостей по дням')
plt.xlabel('День публикации')
plt.ylabel('Кол-во новостей')
fullest_day_img = plt.savefig('fullest_day.png', facecolor = 'lightgrey') #сохраняем как картинку
plt.show()

#КЛЮЧЕВЫЕ СЛОВА
con = sqlite3.connect('news.db')  # подключение
cur = con.cursor()
id_title_text = """
SELECT news_id, title, full_text, pub_day from texts
"""
cur.execute(id_title_text)
df = pd.read_sql_query(id_title_text, con=conn)
list_of_texts = df[['title','full_text']].values.tolist()
articles_preprocessed = []
for a_text in list_of_texts:
    a_tokens = wordpunct_tokenize(a_text[1])
    a_lemmatized = " ".join([morph.parse(item)[0].normal_form for item in a_tokens])
    articles_preprocessed.append(a_lemmatized)
articles_tfidf = tfidf.fit_transform(articles_preprocessed)
print(f"Матрица на {articles_tfidf.shape[0]} документов и {articles_tfidf.shape[1]} термов")

def get_top_tf_idf_words(tfidf_vector, feature_names, top_n):
    sorted_nzs = np.argsort(tfidf_vector.data)[:-(top_n+1):-1]
    return feature_names[tfidf_vector.indices[sorted_nzs]]
feature_names = np.array(tfidf.get_feature_names_out())

for i, article in enumerate(list_of_texts[:1]):
    if i <= 1830:
        article_vector = articles_tfidf[i, :]
        words = get_top_tf_idf_words(article_vector, feature_names, 5)
        print(article[0])
        print(words)

#САМЫЕ ПОПУЛЯРНЫЕ НОВОСТИ
current_month = time.strftime("%m")
popular_news = """
SELECT texts.news_id, title, views_news, pub_month, link
    FROM texts
        JOIN views ON texts.news_id = views.news_id
        WHERE pub_month = ?
        GROUP BY title
        ORDER BY views_news DESC
        LIMIT 3;
"""
cur.execute(popular_news, (current_month,))
df = pd.read_sql_query(popular_news, params=[current_month], con=con)

# Функция, обрабатывающая команду /help
@bot.message_handler(commands=['help'])
def send_welcome(message):
    bot.send_message(message.chat.id, 'Привет, это бот, анализирующий новости. Нажми команду /start, чтобы запустить бота')

@bot.message_handler(commands = ["start"])
def start(message):
    user_id = message.chat.id
    notes[user_id] = message.text
    keyboard = types.InlineKeyboardMarkup()
    item1 = types.InlineKeyboardButton(text = 'Да', callback_data= 'item1')
    item2 = types.InlineKeyboardButton(text = 'Нет', callback_data= 'item2')
    keyboard.add(item1, item2) # прикрепляем к клавиатуре две кнопки
    bot.send_message(message.chat.id, 'Хочешь посмотреть последние новости и статистику по новостям?', reply_markup = keyboard) # прикрепляем клавиатуру к сообщению

# функция запустится, когда пользователь нажмет на кнопку
@bot.callback_query_handler(func=lambda call: True)
def callback_inline(call):
    if call.message:
        if call.data == 'item1':
#БЛОК С ПОСЛЕДНИМИ НОВОСТЯМИ
            keyboard = types.InlineKeyboardMarkup(row_width = 1)
            item3 = types.InlineKeyboardButton(text = 'Последние новости', callback_data= 'item3')
            item4 = types.InlineKeyboardButton(text = 'Статистика по новостям', callback_data= 'item4')
            keyboard.add(item3, item4) # прикрепляем к клавиатуре две кнопки
            bot.send_message(call.message.chat.id, 'Что ты хочешь от меня получить?', reply_markup = keyboard) # прикрепляем клавиатуру к сообщению

    if call.message:
        if call.data == 'item3':     #Если последние новости
            keyboard = types.InlineKeyboardMarkup()
            item11 = types.InlineKeyboardButton(text = 'Спасибо', callback_data= 'item10')
            #последние три новости
            bot.send_message(call.message.chat.id, f'{title}. Подробнее по ссылке: {url}')
            bot.send_message(call.message.chat.id, f'{title1}. Подробнее по ссылке: {url1}')
            bot.send_message(call.message.chat.id, f'{title2}. Подробнее по ссылке: {url2}')
            keyboard.add(item11)
            bot.send_message(call.message.chat.id, 'Держи последние новости:', reply_markup = keyboard)

    if call.message:
#БЛОК СО СТАТИСТИКОЙ
        if call.data == 'item4': #Если статистика
            markup = types.InlineKeyboardMarkup(row_width = 1)
            item5 = types.InlineKeyboardButton(text = 'Ключевые слова', callback_data= 'item5')
            item6 = types.InlineKeyboardButton(text = 'Сколько новостей запостили в день', callback_data= 'item6')
            item7 = types.InlineKeyboardButton(text = 'Самые популярные новости', callback_data= 'item7')
            item8 = types.InlineKeyboardButton(text = 'Близкие по смыслу новости', callback_data= 'item8')
            markup.add(item5, item6, item7, item8) # прикрепляем к клавиатуре две кнопки
            bot.send_message(call.message.chat.id, 'Выбери вариант статистики', reply_markup = markup) # прикрепляем клавиатуру к сообщению
#КЛЮЧЕВЫЕ СЛОВА. РАБОТАЕТ
        if call.data == 'item5': #Если Ключевые слова
            bot.send_message(call.message.chat.id, article[0])
            chat_id = call.message.chat.id
            bot.send_message(chat_id, '\n'.join(map(str, words)))

        if call.data == 'item6': #Если насыщенный день
#Самый насыщенный новостями день.
            img_to_user = open('fullest_day.png', 'rb')
            bot.send_photo(call.message.chat.id, img_to_user)

#ТРИ ПОПУЛЯРНЫЕ НОВОСТИ ЗА МЕСЯЦ ПО КОЛ_ВУ ПРОСМОТРОВ.
        if call.data == 'item7':
            print("Топ-3 самых популярных новостей за текущий месяц:")
            for i, row in df.iterrows(): #перебираю строки в df
                popular ="{}. {} ({})".format(i+1, row['title'], row['link'])
                print(popular)
                bot.send_message(call.message.chat.id, (popular))

#БЛИЗКИЕ НОВОСТИ. РАБОТАЕТ
        if call.data == 'item8': #Близкие новости
            #Близость между векторами текстов, три похожих текста
            list_of_close = []
            list_of_close1 = []
            list_of_close2 = []
            print('Тексты, для которых находят похожие тексты:',list_of_texts[0][0])
            bot.send_message(call.message.chat.id, f'Новость:{list_of_texts[0][0]}')
            for d in cosine_similarity(articles_tfidf[0], articles_tfidf).flatten().argsort()[:-4:-1]:
                list_of_close.append(list_of_texts[d][0])
                print('Тексты:')
            bot.send_message(call.message.chat.id, 'Три похожие новости:')
            for i in list_of_close:
                close = '\n'+(str(i))
                print(close)
                bot.send_message(call.message.chat.id, close)

    if call.message:
#ЦИКЛ
        if call.data == 'item2': #Если "нет"
            markup = types.InlineKeyboardMarkup(row_width = 1)
            item9 = types.InlineKeyboardButton(text = 'Да, я не хочу смотреть новости', callback_data= 'item9')
            item10 = types.InlineKeyboardButton(text = 'Я передумал(а). Хочу смотреть новости!', callback_data= 'item10')
            markup.add(item9, item10)
            bot.send_message(call.message.chat.id, 'Точно?',  reply_markup = markup)

        if call.data == 'item10': #Если передумал
            keyboard = types.InlineKeyboardMarkup(row_width = 1)
            item3 = types.InlineKeyboardButton(text = 'Последние новости', callback_data= 'item3')
            item4 = types.InlineKeyboardButton(text = 'Статистика по новостям', callback_data= 'item4')
            keyboard.add(item3,item4)
            bot.send_message(call.message.chat.id, 'Что ты хочешь от меня получить?', reply_markup = keyboard)

        if call.data == 'item9': #Если не передумал
            bot.send_message(call.message.chat.id, 'Пока')

        if call.data in ['item5', 'item6', 'item7', 'item8', 'item11']: #Цикл после того, как прислалась статистика или последние новости
            keyboard = types.InlineKeyboardMarkup()
            item1 = types.InlineKeyboardButton(text = 'Да', callback_data= 'item1')
            item2 = types.InlineKeyboardButton(text = 'Нет', callback_data= 'item2')
            keyboard.add(item1, item2) # прикрепляем к клавиатуре две кнопки
            bot.send_message(call.message.chat.id, 'Хочешь посмотреть последние новости и статистику по новостям?', reply_markup = keyboard)  # прикрепляем клавиатуру к сообщению

bot.polling(none_stop=True)

# пустая главная страничка для проверки
@app.route('/', methods=['GET', 'HEAD'])
def index():
    return 'ok'

# обрабатываем вызовы вебхука = функция, которая запускается, когда к нам постучался телеграм
@app.route(WEBHOOK_URL_PATH, methods=['POST'])
def webhook():
    if flask.request.headers.get('content-type') == 'application/json':
        json_string = flask.request.get_data().decode('utf-8')
        update = telebot.types.Update.de_json(json_string)
        bot.process_new_updates([update])
        return ''
    else:
        flask.abort(403)


from .models import ServicePriceTable, CostCenter, PolicyPrice, BillingMoment
from rest_framework import serializers
from .utils import valid_court_district, check_service_price_table_unique, check_office_correspondent_relation
from core.serializers import CreateUserSerializerMixin, OfficeSerializerMixin, OfficeSerializer, PersonSerializer, \
    AuditSerializerMixin
from lawsuit.serializers import CourtDistrictSerializer, CourtDistrictComplementSerializer
from task.serializers import TypeTaskSerializer

class PolicyPriceSerializer(serializers.ModelSerializer, AuditSerializerMixin, CreateUserSerializerMixin,
                            OfficeSerializerMixin):
    class Meta:
        model = PolicyPrice
        exclude = ('create_date', 'alter_date', 'alter_user')

class ServicePriceTableSerializer(serializers.ModelSerializer, AuditSerializerMixin, CreateUserSerializerMixin,
                                  OfficeSerializerMixin):

    class Meta:
        model = ServicePriceTable
        exclude = ('create_date', 'alter_date', 'alter_user', 'value_to_receive_currency', 'value_to_receive',
                   'value_to_pay_currency', 'value_to_pay', 'rate_type_pay', 'rate_type_receive')

    def validate(self, data):
        pk = self.instance.id if self.instance else None
        office = data.get('office')
        court_district = data.get('court_district')
        court_dictrict_complement = data.get('court_district_complement')
        state = data.get('state')
        office_correspondent = data.get('office_correspondent')
        client = data.get('client')
        type_task = data.get('type_task')
        if office_correspondent:
            if not check_office_correspondent_relation(office, office_correspondent):
                raise serializers.ValidationError("O escritório correspondente informado não possui relacionamento com "
                                                  "o escritório da sessão")
        if court_district and state:
            if not valid_court_district(court_district, state):
                raise serializers.ValidationError("A Comarca selecionada não pertence à UF selecionada")
        if not check_service_price_table_unique(pk, office, office_correspondent, state, court_district, type_task,
                                                client, court_dictrict_complement):
            raise serializers.ValidationError("Os campos office, office_correspondent, type_task, client, "
                                              "court_district, state e court_district_complement devem criar "
                                              "um set único.")
        return data

class ServicePriceDelegationTableSerializer(serializers.ModelSerializer,
                                            CreateUserSerializerMixin, OfficeSerializerMixin):
    policy_price = PolicyPriceSerializer(many=False, read_only=True)
    office_correspondent = OfficeSerializer(many=False, read_only=True)
    type_task = TypeTaskSerializer(many=False, read_only=True)
    office_network = serializers.SerializerMethodField()
    state = serializers.SerializerMethodField()
    court_district = CourtDistrictSerializer(many=False, read_only=True)
    court_district_complement = CourtDistrictComplementSerializer(many=False, read_only=True)
    city = serializers.SerializerMethodField()
    client = PersonSerializer(many=False, read_only=True)
    office_rating = serializers.SerializerMethodField()
    finished_by_rate = serializers.SerializerMethodField()
    office_return_rating = serializers.SerializerMethodField()

    class Meta:
        model = ServicePriceTable
        exclude = ('create_date', 'alter_date', 'alter_user')

    def get_office_network(self, obj):
        return obj.office_network.name if obj.office_network else ''

    def get_state(self, obj):
        return obj.state.initials if obj.state else None

    def get_city(self, obj):
        return obj.city.name if obj.city else None

    def get_office_rating(self, obj):
        return obj.office_rating

    def get_finished_by_rate(self, obj):
        return obj.finished_by_rate

    def get_office_return_rating(self, obj):
        return obj.office_return_rating

    def validate(self, data):
        pk = self.instance.id if self.instance else None
        office = data.get('office')
        court_district = data.get('court_district')
        court_dictrict_complement = data.get('court_district_complement')
        state = data.get('state')
        office_correspondent = data.get('office_correspondent')
        client = data.get('client')
        type_task = data.get('type_task')
        if office_correspondent:
            if not check_office_correspondent_relation(office, office_correspondent):
                raise serializers.ValidationError("O escritório correspondente informado não possui relacionamento com "
                                                  "o escritório da sessão")
        if court_district and state:
            if not valid_court_district(court_district, state):
                raise serializers.ValidationError("A Comarca selecionada não pertence à UF selecionada")
        if not check_service_price_table_unique(pk, office, office_correspondent, state, court_district, type_task,
                                                client, court_dictrict_complement):
            raise serializers.ValidationError("Os campos office, office_correspondent, type_task, client, "
                                              "court_district, state e court_district_complement devem criar "
                                              "um set único.")
        return data

class CostCenterSerializer(serializers.ModelSerializer, AuditSerializerMixin, CreateUserSerializerMixin,
                           OfficeSerializerMixin):

    class Meta:
        model = CostCenter
        exclude = ('create_date', 'alter_date', 'alter_user')


from django.core.management.base import BaseCommand

import catalog.models
from catalog.models import Users
from django.contrib.auth.models import User

class Command(BaseCommand):

    def add_arguments(self, parser):
        parser.add_argument('arg_1', nargs='+', type=int)

    def handle(self, *args, **options):
        q = User.objects.filter(id__in=options['arg_1'])
        if q.filter(is_superuser=True).exists():
            self.stdout.write(self.style.SUCCESS('There is one or more superusers so I cant delete this items'))
        else:
            q.delete()
            self.stdout.write(self.style.SUCCESS('Items was successfully deleted'))