from __future__ import (
    absolute_import,
    unicode_literals,
    with_statement,
)

__all__ = [
    'DoctestReviewer',
]

import _ast
import os
import re
import sys
from difflib import unified_diff
from doctest import DocTestParser, Example
from optparse import OptionParser
from textwrap import wrap

from pocketlint.reporter import Reporter
try:
    from pyflakes.checker import Checker as PyFlakesChecker
    PyFlakesChecker
except ImportError:
    from pocketlint import PyFlakesChecker

class DoctestReviewer(object):
    """Check and reformat doctests."""
    rule_pattern = re.compile(r'([=~-])+[ ]*$')
    moin_pattern = re.compile(r'^(=+)[ ](.+)[ ](=+[ ]*)$')

    SOURCE = 'source'
    WANT = 'want'
    NARRATIVE = 'narrative'

    def __init__(self, file_path, doctest, reporter=None, options=None):
        self.doctest = doctest
        self.file_path = file_path
        self.base_dir = os.path.dirname(file_path)
        self.file_name = os.path.basename(file_path)
        self.blocks = []
        self.block = []
        self.block_method = self.preserve_block
        self.code_lines = []
        self.example = None
        self.last_bad_indent = 0
        self.has_printed_filename = False
        self._reporter = reporter or Reporter(Reporter.CONSOLE)
        self.options = options

    def get_parts(self):
        parser = DocTestParser()
        try:
            return parser.parse(self.doctest, self.file_path)
        except ValueError as error:
            # Output code without unicode literals needs to be normalised
            # largely for the test suite, and somewhat for the person reading
            # message.
            message = str(error).replace("u'", "'")
            self._print_message(message, 0)
            return []

    def _print_message(self, message, lineno):
        """Print the error message with the lineno.

        :param message: The message to print.
        :param lineno: The line number the message pertains to.
        """
        self._reporter(
            int(lineno), message,
            base_dir=self.base_dir, file_name=self.file_name)

    def _is_formatted(self, text):
        """Return True if the text is pre-formatted, otherwise False.

        :param: text a string, or a list of strings.
        """
        if isinstance(text, list):
            text = text[0]
        return text.startswith(' ')

    def _walk(self, doctest_parts):
        """Walk the doctest parts; yield the line and kind.

        Yield the content of the line, and its kind (SOURCE, WANT, NARRATIVE).
        SOURCE and WANT lines are stripped of indentation, SOURCE is also
        stripped of the interpreter symbols.

        :param doctest_parts: The output of DocTestParser.parse.
        """
        for part in doctest_parts:
            if part == '':
                continue
            if isinstance(part, Example):
                self.example = part
                for line in part.source.splitlines():
                    kind = DoctestReviewer.SOURCE
                    yield line, kind
                for line in part.want.splitlines():
                    kind = DoctestReviewer.WANT
                    yield line, kind
            else:
                self.example = None
                kind = DoctestReviewer.NARRATIVE
                for line in part.splitlines():
                    yield line, kind

    def _apply(self, line_methods):
        """Call each line_method for each line in the doctest.

        :param line_methods: a list of methods that accept lineno, line,
            and kind as arguments. Each method must return the line for
            the next method to process.
        """
        self.blocks = []
        self.block = []
        lineno = 0
        previous_kind = DoctestReviewer.NARRATIVE
        for line, kind in self._walk(self.get_parts()):
            lineno += 1
            # Some method could check if the line number is one that
            # is skipped by doctest parser and increment the number again.
            # line probably needs to get a \n add to it too.
            self._append_source(kind, line)
            if kind != previous_kind and kind != DoctestReviewer.WANT:
                # The WANT block must adjoin the preceding SOURCE block.
                self._store_block(previous_kind)
            for method in line_methods:
                line = method(lineno, line, kind, previous_kind)
                if line is None:
                    break
            if not line:
                continue
            self.block.append(line)
            previous_kind = kind
        # Capture the last block and a blank line.
        self.block.append('\n')
        self._store_block(previous_kind)

    def _append_source(self, kind, line):
        """Update the list of source code lines seen."""
        if kind == self.SOURCE:
            self.code_lines.append(line)
        else:
            self.code_lines.append('')

    def _store_block(self, kind):
        """Append the block to blocks, re-wrap unformatted narrative.

        :param kind: The block's kind (SOURCE, WANT, NARRATIVE)
        """
        if len(self.block) == 0:
            return
        block = self.block_method(kind, self.block, self.blocks)
        self.blocks.append('\n'.join(block))
        self.block = []

    def check(self):
        """Check the doctest for style and code issues.

        1. Check line lengths.
        2. Check that headings are not in Moin format.
        3. Check indentation.
        4. Check trailing whitespace.
        """
        self.code_lines = []
        self.last_bad_indent = 0
        self.block_method = self.preserve_block
        self.example = None
        self.has_printed_filename = False
        self.check_source_comments()
        line_checkers = [
            self.check_length,
            self.check_heading,
            self.check_indentation,
            self.check_trailing_whitespace,
        ]
        self._apply(line_checkers)
        code = '\n'.join(self.code_lines)
        self.check_source_code(code)

    def format(self):
        """Reformat doctest.

        1. Tests are reindented to 4 spaces.
        2. Simple narrative is rewrapped to 78 character width.
        3. Formatted (indented) narrative is preserved.
        4. Moin headings are converted to RSR =, == , and === levels.
        5. There is one blank line between blocks,
        6. Except for headers which have two leading blank lines.
        7. All trailing whitespace is removed.

        SOURCE and WANT long lines are not fixed--this is a human operation.
        """
        self.code_lines = []
        line_checkers = [
            self.fix_trailing_whitespace,
            self.fix_indentation,
            self.fix_heading,
            self.fix_narrative_paragraph,
        ]
        self.block_method = self.format_block
        self._apply(line_checkers)
        self.block_method = self.preserve_block
        return '\n\n'.join(self.blocks)

    def preserve_block(self, kind, block, blocks):
        """Do nothing to the block.

        :param kind: The block's kind (SOURCE, WANT, NARRATIVE)
        :param block: The list of lines that should remain together.
        :param blocks: The list of all collected blocks.
        """
        return block

    def format_block(self, kind, block, blocks):
        """Format paragraph blocks.

        :param kind: The block's kind (SOURCE, WANT, NARRATIVE)
        :param block: The list of lines that should remain together.
        :param blocks: The list of all collected blocks.
        """
        if kind != DoctestReviewer.NARRATIVE or self._is_formatted(block):
            return block
        try:
            rules = ('===', '---', '...')
            last_line = block[-1]
            is_heading = last_line[0:3] in rules and last_line[-3:] in rules
        except IndexError:
            is_heading = False
        if len(blocks) != 0 and is_heading:
            # Headings should have an extra leading blank line.
            block.insert(0, '')
        elif is_heading:
            # Do nothing. This is the first heading in the file.
            pass
        else:
            long_line = ' '.join(block).strip()
            block = wrap(long_line, 72)
        return block

    def check_source_comments(self):
        """Comments are not appropiate in source examples."""
        for lineno, line in enumerate(self.doctest.splitlines()):
            if '>>> #' in line or '... #' in line:
                self._print_message(
                    'Comment belongs in narrative.', lineno + 1)

    def is_code_comment(self, line):
        """Return True if the line is a code comment."""
        comment_pattern = re.compile(r'^\s+#')
        return comment_pattern.match(line) is not None

    def check_length(self, lineno, line, kind, previous_kind):
        """Check the length of the line.

        Each kind of line has a maximum length:

        * NARRATIVE: 78 characters.
        * SOURCE: 70 characters (discounting indentation and interpreter).
        * WANT: 74 characters (discounting indentation).
        """
        if kind == DoctestReviewer.NARRATIVE and self.is_code_comment(line):
            # comments follow WANT rules because they are in code.
            kind = DoctestReviewer.WANT
            line = line.lstrip()
        length = len(line)
        if kind == DoctestReviewer.NARRATIVE and length > 78:
            self._print_message('%s exceeds 78 characters.' % kind, lineno)
        elif kind == DoctestReviewer.WANT and length > 74:
            self._print_message('%s exceeds 78 characters.' % kind, lineno)
        elif kind == DoctestReviewer.SOURCE and length > 70:
            self._print_message('%s exceeds 78 characters.' % kind, lineno)
        else:
            # This line has a good length.
            pass
        return line

    def check_indentation(self, lineno, line, kind, previous_kind):
        """Check the indentation of the SOURCE or WANT line."""
        if kind == DoctestReviewer.NARRATIVE:
            return line
        if self.example.indent != 4:
            if self.last_bad_indent != lineno - 1:
                self._print_message('%s has bad indentation.' % kind, lineno)
            self.last_bad_indent = lineno
        return line

    def check_trailing_whitespace(self, lineno, line, kind, previous_kind):
        """Check for the presence of trailing whitespace in the line."""
        if line.endswith(' '):
            self._print_message('%s has trailing whitespace.' % kind, lineno)
        return line

    def check_heading(self, lineno, line, kind, previous_kind):
        """Check for narrative lines that use moin headers instead of RST."""
        if kind != DoctestReviewer.NARRATIVE:
            return line
        moin = self.moin_pattern.match(line)
        if moin is not None:
            self._print_message('%s uses a moin header.' % kind, lineno)
        return line

    def check_source_code(self, code):
        """Check for source code problems in the doctest using pyflakes.

        The most common problem found are unused imports. `UndefinedName`
        errors are suppressed because the test setup is not known.
        """
        if code == '':
            return
        try:
            tree = compile(
                code, self.file_path, "exec", _ast.PyCF_ONLY_AST)
        except (SyntaxError, IndentationError) as exc:
            lineno = exc.lineno or 0
            line = exc.text or ''
            if line.endswith("\n"):
                line = line[:-1]
            self._print_message(
                'Could not compile:\n    %s' % line, lineno)
        else:
            warnings = PyFlakesChecker(tree)
            for warning in warnings.messages:
                if 'undefined name ' in str(warning):
                    continue
                dummy, lineno, message = str(warning).split(':')
                self._print_message(message.strip(), lineno)

    def fix_trailing_whitespace(self, lineno, line, kind, previous_kind):
        """Return the line striped of trailing whitespace."""
        return line.rstrip()

    def fix_indentation(self, lineno, line, kind, previous_kind):
        """set the indentation to 4-spaces."""
        if kind == DoctestReviewer.NARRATIVE:
            return line
        elif kind == DoctestReviewer.WANT:
            return '    %s' % line
        else:
            if line.startswith(' '):
                # This is a continuation of DoctestReviewer.SOURCE.
                return '    ... %s' % line
            else:
                # This is a start of DoctestReviewer.SOURCE.
                return '    >>> %s' % line

    def fix_heading(self, lineno, line, kind, previous_kind):
        """Switch Moin headings to RST headings."""
        if kind != DoctestReviewer.NARRATIVE:
            return line
        moin = self.moin_pattern.match(line)
        if moin is None:
            return line
        heading_level = len(moin.group(1))
        heading = moin.group(2)
        rule_length = len(heading)
        if heading_level == 1:
            rule = '=' * rule_length
        elif heading_level == 2:
            rule = '-' * rule_length
        else:
            rule = '.' * rule_length
        # Force the heading on to the block of lines.
        self.block.append(heading)
        return rule

    def fix_narrative_paragraph(self, lineno, line, kind, previous_kind):
        """Break narrative into paragraphs."""
        if kind != DoctestReviewer.NARRATIVE or len(self.block) == 0:
            return line
        if line == '':
            # This is the start of a new paragraph in the narrative.
            self._store_block(previous_kind)
        if self._is_formatted(line) and not self._is_formatted(self.block):
            # This line starts a pre-formatted paragraph.
            self._store_block(previous_kind)
        return line

    def format_and_save(self, is_interactive=False):
        new_doctest = self.format()
        if new_doctest != self.doctest:
            if is_interactive:
                diff = unified_diff(
                    self.doctest.splitlines(), new_doctest.splitlines())
                sys.stdout.write('\n'.join(diff))
                sys.stdout.write('\n\n')
                if sys.version_info >= (3,):
                    user_input = input
                else:
                    user_input = raw_input
                do_save = user_input(
                    'Do you wish to save the changes? S(ave) or C(ancel)?')
            else:
                do_save = 'S'
            self.doctest = new_doctest
            if do_save.upper() == 'S':
                with open(self.file_path, 'wt') as doctest_file:
                    doctest_file.write(new_doctest)

def get_option_parser():
    """Return the option parser for this program."""
    usage = "usage: %prog [options] doctest.txt"
    parser = OptionParser(usage=usage)
    parser.add_option(
        "-f", "--format", dest="is_format", action="store_true",
        help="Reformat the doctest.")
    parser.add_option(
        "-i", "--interactive", dest="is_interactive", action="store_true",
        help="Approve each change.")
    parser.set_defaults(
        is_format=False,
        is_interactive=False)
    return parser

def main(argv=None):
    """Run the operations requested from the command line."""
    if argv is None:
        argv = sys.argv
    parser = get_option_parser()
    (options, args) = parser.parse_args(args=argv[1:])
    if len(args) == 0:
        parser.error("A doctest must be specified.")

    for file_path in args:
        with open(file_path) as doctest_file:
            doctest_data = doctest_file.read().decode('utf-8')
        reviewer = DoctestReviewer(file_path, doctest_data)
        if options.is_format:
            reviewer.format_and_save()
        reviewer.check()

if __name__ == '__main__':
    sys.exit(main())


import MDAnalysis as mda
from rdkit import Chem
from rdkit.Chem import AllChem

"""Useful for SMILES to Vina"""

# http://chemyang.ccnu.edu.cn/ccb/server/AIMMS/mol2.pdf ##see for more explanations for SYBYL!
# https://github.com/jensengroup/xyz2mol/blob/master/xyz2mol.py ## see for better conversion!

############
#####WIP####
############

def filter_mol2_atoms(at: "MDAnalysis atom name", atom: "RDKIT mol atom instance"):
    if at in ["C", "N", "O", "S", "P"]:
        if atom.GetHybridization().name.lower() == "sp3":
            return ".3"
        elif atom.GetHybridization().name.lower() == "sp2":
            return ".2"
        elif atom.GetHybridization().name.lower() == "sp":
            return ".1"
        elif atom.GetIsAromatic():
            return ".ar"
    else:
        return ""

def filter_mol2_bonds(bo: "MDAnalysis order"):
    if bo in [1, 2, 3]:
        return int(bo)
    else:
        return "ar"

def encode_block(filename, mol):
    """
    Parameters
    ----------
    obj : AtomGroup or Universe
    """
    try:
        mol.GetConformer()
    except Exception as e:
        print(e)
        mol = Chem.AddHs(mol)
        AllChem.EmbedMoecule(mol)

    obj = mda.Universe(mol)
    # Issue 2717
    obj = obj.atoms
    traj = obj.universe.trajectory
    ts = traj.ts

    molecule = []
    # try:
    #     molecule = ts.data['molecule']
    # except KeyError:
    #     raise_from(NotImplementedError(
    #         "MOL2Writer cannot currently write non MOL2 data"),
    #         None)

    # Need to remap atom indices to 1 based in this selection
    mapping = {a: i for i, a in enumerate(obj.atoms, start=1)}

    # Grab only bonds between atoms in the obj
    # ie none that extend out of it
    bondgroup = obj.bonds.atomgroup_intersection(obj, strict=True)
    bonds = sorted((b[0], b[1], b.order) for b in bondgroup)
    bond_lines = ["@<TRIPOS>BOND"]
    bond_lines.extend("{0:>5} {1:>5} {2:>5} {3:>2}"
                      "".format(bid,
                                mapping[atom1],
                                mapping[atom2],
                                filter_mol2_bonds(order))
                      for bid, (atom1, atom2, order)in enumerate(
                              bonds, start=1))
    bond_lines.append("\n")
    bond_lines = "\n".join(bond_lines)

    atom_lines = ["@<TRIPOS>ATOM"]
    atom_lines.extend("{0:>4} {1:>4} {2:>13.4f} {3:>9.4f} {4:>9.4f}"
                      " {5:>4} {6} {7} {8:>7.4f}"
                      "".format(mapping[a],
                                a.name,
                                a.position[0],
                                a.position[1],
                                a.position[2],
                                a.type[0] + filter_mol2_atoms(a.type[0], at), ######WIP!
                                a.resid,
                                # a.resname,
                                # a.charge)
                                "lig",
                                0)
                      for a, at in zip(obj.atoms, mol.GetAtoms()))
    atom_lines.append("\n")
    atom_lines = "\n".join(atom_lines)

    # try:
    #     substructure = ["@<TRIPOS>SUBSTRUCTURE\n"] + ts.data['substructure']
    # except KeyError:
    #     substructure = ""
    substructure = ["@<TRIPOS>SUBSTRUCTURE\n"] + ["1 **** 1 TEMP 0 **** **** 0 ROOT"]

    # check_sums = molecule[1].split()
    # check_sums[0], check_sums[1] = str(len(obj.atoms)), str(len(bondgroup))

    # prevent behavior change between repeated calls
    # see gh-2678
    # molecule_0_store = molecule[0]
    # molecule_1_store = molecule[1]

    # molecule[1] = "{0}\n".format(" ".join(check_sums))

    molecule.insert(0, "@<TRIPOS>MOLECULE\n")
    molecule.insert(1, "LIG\n")
    molecule.insert(2, f"{len(obj.atoms)} {len(bonds)} {0} {0} {1}\n")
    molecule.insert(3, "SMALL\n")
    molecule.insert(4, "USER_CHARGES\n\n")

    return_val = ("".join(molecule) + atom_lines +
                  bond_lines + "".join(substructure))

    # molecule[0] = molecule_0_store
    # molecule[1] = molecule_1_store

    f = open(filename, "w")
    f.write(return_val)

    return return_val


import torch

def get_smooth_weights(losses, spectrum, smooth_coef, smoothing='l2'):
    """
    Losses are the values of the losses at the current iterate, spectrum are the weights of the spectral measure
    considered given in non-decreasing order
    :param losses: (torch.Tensor of shape (n,) values of the losses at the current iterate
    :param spectrum: (torch.Tensor of shape (n,) weights of the spectral measure considered given in non-decreasing
    order
    :param smooth_coef: (float) value of the smoothing coefficient
    :param smoothing: (str) choose between 'l2' and 'neg_entropy' for resulting weights that are either
    smooth w.r.t. l2 norm or l1 norm, see latex notes for more details (note that we use centered smoothing operators)
    :return:
    """
    n = len(losses)
    scaled_losses = losses/smooth_coef
    perm = torch.argsort(losses)
    sorted_losses = scaled_losses[perm]

    if smoothing == 'l2':
        primal_sol = l2_centered_isotonic_regression(sorted_losses, spectrum)
    elif smoothing == 'neg_entropy':
        primal_sol = neg_entropy_centered_isotonic_regression(sorted_losses, spectrum)
    else:
        raise NotImplementedError
    inv_perm = torch.argsort(perm)
    primal_sol = primal_sol[inv_perm]
    if smoothing == 'l2':
        smooth_weights = scaled_losses - primal_sol + 1/n
    elif smoothing == 'neg_entropy':
        smooth_weights = torch.exp(scaled_losses - primal_sol)/n
    else:
        raise NotImplementedError
    return smooth_weights

def l2_centered_isotonic_regression(losses, spectrum):
    n = len(losses)
    means = [losses[0] + 1/n - spectrum[0]]
    counts = [1]
    end_points = [0]
    for i in range(1, n):
        means.append(losses[i] + 1/n - spectrum[i])
        counts.append(1)
        end_points.append(i)
        while len(means) > 1 and means[-2] >= means[-1]:
            prev_mean, prev_count, prev_end_point = means.pop(), counts.pop(), end_points.pop()
            means[-1] = (counts[-1]*means[-1] + prev_count*prev_mean)/(counts[-1] + prev_count)
            counts[-1] = counts[-1] + prev_count
            end_points[-1] = prev_end_point
    sol = output_sol_iso_reg(end_points, means, n)
    # plt.plot(sol)
    # plt.show()
    return sol

def neg_entropy_centered_isotonic_regression(losses, spectrum):
    n = len(losses)
    logn = torch.log(torch.tensor(n))
    log_spectrum = torch.log(spectrum)

    lse_losses = [losses[0]]
    lse_log_spectrum = [log_spectrum[0]]
    means = [losses[0] - log_spectrum[0] - logn]
    end_points = [0]
    for i in range(1, n):
        means.append(losses[i] - log_spectrum[i] - logn)
        lse_losses.append(losses[i])
        lse_log_spectrum.append(log_spectrum[i])
        end_points.append(i)
        while len(means) > 1 and means[-2] >= means[-1]:
            prev_mean, prev_lse_loss, prev_lse_log_spectrum, prev_end_point = means.pop(), lse_losses.pop(), \
                                                                              lse_log_spectrum.pop(), end_points.pop()
            new_lse_loss = torch.logsumexp(torch.tensor([lse_losses[-1], prev_lse_loss]), dim=0)
            new_lse_log_spectrum = torch.logsumexp(torch.tensor([lse_log_spectrum[-1], prev_lse_log_spectrum]), dim=0)
            means[-1] = new_lse_loss - new_lse_log_spectrum - logn
            lse_losses[-1], lse_log_spectrum[-1] = new_lse_loss, new_lse_log_spectrum
            end_points[-1] = prev_end_point
    sol = output_sol_iso_reg(end_points, means, n)
    # plt.plot(sol)
    # plt.show()
    return sol

def output_sol_iso_reg(end_points, means, n):
    sol = torch.zeros(n)
    i = 0
    for j in range(len(end_points)):
        end_point = end_points[j]
        sol[i:end_point + 1] = means[j]
        i = end_point + 1
    return sol

if __name__ == '__main__':
    n = 1000
    smooth_coef = 0.1
    # Try with extremile coefficients below
    r = 5
    spectrum = ((torch.arange(n, dtype=torch.float64) + 1) ** r - torch.arange(n, dtype=torch.float64) ** r) / (n ** r)
    for i in range(20):
        losses = torch.randn(n, dtype=torch.float64)
        perm = torch.argsort(losses)
        invperm = torch.argsort(perm)

        # l2 smoothing
        # The right scaling for the l2 smoothing should be n times the smoothing coefficient see notes
        smooth_weights = get_smooth_weights(losses, spectrum, n*smooth_coef, smoothing='l2')
        print('Sum smooth weights l2 smoothing (should be 1):{}'.format(torch.sum(smooth_weights)))
        print('Norm diff btw non-smooth & smoothed weights l2 smoothing:{}'.format(torch.norm(
            smooth_weights-spectrum[invperm])))

        # Negative entropy smoothing
        smooth_weights = get_smooth_weights(losses, spectrum, smooth_coef, smoothing='neg_entropy')
        print('Sum smooth weights neg ent smoothing (should be 1):{}'.format(torch.sum(
            smooth_weights)))
        print('Norm diff btw non-smooth & smoothed weights neg ent smoothing:{}'.format(torch.norm(
            smooth_weights-spectrum[invperm])))

    # Try with erm, i.e., uniform spectrum, should give us smooth_weights = uniform
    smooth_coef = 0.1
    spectrum = torch.ones(n)/n
    for i in range(20):
        losses = torch.randn(n, dtype=torch.float64)
        smooth_weights = get_smooth_weights(losses, spectrum, n*smooth_coef, smoothing='l2')
        print("Norm diff l2 smooth weights uniform (should be 0):{}".format(torch.norm(
            spectrum-smooth_weights)))
        smooth_weights = get_smooth_weights(losses, spectrum, n * smooth_coef, smoothing='neg_entropy')
        print("Norm diff neg ent smooth weights uniform (should be 0):{}".format(torch.norm(
            spectrum - smooth_weights)))


#!/usr/bin/env python3
"""retroarcher.py

Responsible for starting RetroArcher.
"""
# future imports
from __future__ import annotations

# standard imports
import argparse
import os
import sys
import time

# local imports
import pyra
from pyra import config
from pyra import definitions
from pyra import logger
from pyra import webapp

py_name = 'pyra'

class IntRange(object):
    """Custom IntRange class for argparse.

    Prevents printing out large list of possible choices for integer ranges.

    Raises argparse.ArgumentTypeError if provided value is outside the accepted range.
    """
    def __init__(self, stop: int, start: int = 0,):
        """Initialize the IntRange class object.

        If stop is less than start, the values will be corrected automatically.

        :param stop: int - Range maximum value (required)
        :param start: int - Range minimum value (optional). Default = 0
        """
        if stop < start:
            stop, start = start, stop
        self.start, self.stop = start, stop

    def __call__(self, value: int | str) -> int:
        """Validates value is within accepted range.

        :param value: int | str - The value to validate.
        :return: int - Returned if value is valid, otherwise argparse.ArgumentTypeError is raised.
        """
        value = int(value)
        if value < self.start or value >= self.stop:
            raise argparse.ArgumentTypeError(f'Value outside of range: ({self.start}, {self.stop})')
        return value

def main():
    """Application entry point.

    Parses arguments and initializes the application.
    """
    # Fixed paths to RetroArcher
    if hasattr(sys, 'frozen') and hasattr(sys, '_MEIPASS'):
        pyra.FROZEN = True

        import pyi_splash  # module cannot be installed and is only available when using the pyinstaller build
        pyi_splash.update_text("Attempting to start RetroArcher")

    # Set up and gather command line arguments
    parser = argparse.ArgumentParser(description='RetroArcher is a Python based game streaming server.\n'
                                                 'Arguments supplied here are meant to be temporary.')

    parser.add_argument('--config', help='Specify a config file to use')
    parser.add_argument('--debug', action='store_true', help='Use debug logging level')
    parser.add_argument('--dev', action='store_true', help='Start RetroArcher in the development environment')
    parser.add_argument(
        '-p', '--port', default=9696, type=IntRange(21, 65535),
        help='Force RetroArcher to run on a specified port, default=9696'
    )
    parser.add_argument('-q', '--quiet', action='store_true', help='Turn off console logging')
    parser.add_argument(
        '-v', '--version', action='store_true',
        help='Print the version details and exit.'
    )

    args = parser.parse_args()

    if args.version:
        print('todo_v')  # todo
        SystemExit()

    if args.config:
        config_file = args.config
    else:
        config_file = os.path.join(definitions.Paths().DATA_DIR, definitions.Files().CONFIG)
    if args.debug:
        pyra.DEBUG = True
    if args.dev:
        pyra.DEV = True
    if args.quiet:
        pyra.QUIET = True

    # initialize retroarcher
    pyra.initialize(config_file=config_file)

    # get logger
    log = logger.get_logger(name=py_name)

    if args.config:
        log.info(msg=f"RetroArcher is using custom config file: {config_file}.")
    if args.debug:
        log.info(msg="RetroArcher will log debug messages.")
    if args.dev:
        log.info(msg="RetroArcher is running in the dev environment.")
    if args.quiet:
        log.info(msg="RetroArcher is running in quiet mode. Nothing will be printed to console.")

    if args.port != 9696:
        config.CONFIG['Network']['HTTP_PORT'] = args.port
        config.CONFIG.write()

    # start the webapp
    if pyra.FROZEN:  # pyinstaller build only
        pyi_splash.update_text("Starting the webapp")
        time.sleep(3)  # show splash screen for a min of 3 seconds
        pyi_splash.close()  # close the splash screen
    webapp.app.run(
        host=config.CONFIG['Network']['HTTP_HOST'],
        port=config.CONFIG['Network']['HTTP_PORT'],
        debug=pyra.DEV
    )

if __name__ == "__main__":
    main()


from numpy import array
from basicLSTMs import get_stacked_lstm
from getSeqs import in_seq1 as raw_seq
from keras.models import Sequential
from keras.layers import LSTM, Dense, RepeatVector, TimeDistributed

EncoderDecoder = True

def get_encdec_model(n_steps_in, n_features, n_steps_out, n_units=100):
    model = Sequential()
    model.add(LSTM(n_units, activation='relu', input_shape=(n_steps_in, n_features)))
    model.add(RepeatVector(n_steps_out))
    model.add(LSTM(n_units, activation='relu', return_sequences=True))
    model.add(TimeDistributed(Dense(n_features)))
    model.compile(optimizer='adam', loss='mse')
    return model

# split a univariate sequence into samples
def split_sequence(sequence, n_steps_in, n_steps_out):
    X, y = list(), list()
    for i in range(len(sequence)):
        # find the end of this pattern
        end_ix = i + n_steps_in
        out_end_ix = end_ix + n_steps_out
        # check if we are beyond the sequence
        if out_end_ix > len(sequence):
            break
        # gather input and output parts of the pattern
        seq_x, seq_y = sequence[i:end_ix], sequence[end_ix:out_end_ix]
        X.append(seq_x)
        y.append(seq_y)
    return array(X), array(y)

if __name__ == '__main__':
    # choose a number of time steps
    n_steps_in, n_steps_out = 3, 2
    # split into samples
    X, y = split_sequence(raw_seq, n_steps_in, n_steps_out)
    # reshape from [samples, timesteps] into [samples, timesteps, features]
    n_features = 1
    X = X.reshape((X.shape[0], X.shape[1], n_features))
    if EncoderDecoder:
        y = y.reshape((y.shape[0], y.shape[1], n_features))
        model = get_encdec_model(n_steps_in, n_features, n_steps_out)
    else:
        model = get_stacked_lstm(n_steps_in, n_features, n_outputs=n_steps_out)
    # fit model
    model.fit(X, y, epochs=200, verbose=1)
    # demonstrate prediction
    x_input = array([70, 80, 90])
    x_input = x_input.reshape((1, n_steps_in, n_features))
    yhat = model.predict(x_input, verbose=0)
    print(yhat)


# The project was implemented with the Python programming language Izhikevich Neuron Model, which provides a real-time audio response to the input audio signal.
# This system processes the input audio signal and transfers it to the Izhikevich Neuron Model algorithm and simultaneously responds with the input audio signal.
import pyaudio
import numpy as np
from scipy import signal
from multiprocessing import Process, Queue
from threading import Thread
import matplotlib.pyplot as plt
import time

winSize = 10   # time the CHUNK
CHUNK = 1024
FORMAT = pyaudio.paFloat32
CHANNELS = 1
RATE = 44100
h = 0.2
# Values of 10 as the window size and 1024 as the number of continuously changing samples are given.
# The audio signal was recorded in float32 and worked on a single channel.
# The number of samples per second is set to 44100.
# A value of 0.2 is defined for subsampling and the Izhikevich algorithm.

# The processor is the processor that performs the actions that start and finish the algorithm. Two operations were performed within a single processor.
# One of these two actions continuously records the sound from the microphone, the other processes the last 1024 samples and transmits it to the speaker as an Izhikevich response.
# To minimize the latency that may occur, these two separate operations were performed on the same processor and the refresh frequency was set to 1024 (optimal value).
# 1024 new incoming samples were added in succession continuously and used in the audio signal processing algorithm to process it.
# By converting numeric values to "byte" size, I got the output.

# I recorded continuous audio with the callback function.
# Values retrieved in float32 are given as 'in_data' (input data).
# As 'out_data' (output data), the converted values in bytes of the Izhikevich response, or the byte equivalent of 0, are given on a case-by-case basis whether the 'izi2record' queue is empty.
# Thus, I performed audio recording and audio output.

def recordd(record2soundprocess, izi2record):
    p = pyaudio.PyAudio()

    def callback(in_data, frame_count, time_info, status):
        # to process
        incoming = np.frombuffer(in_data, np.float32);
        record2soundprocess.put(incoming)

        # From izi
        if izi2record.empty():
            out_data = np.zeros(np.shape(incoming)).tobytes();
        else:
            incoming = izi2record.get()
            out_data = incoming.tobytes();

        return (out_data, pyaudio.paContinue)

    stream = p.open(format=FORMAT,
                    channels=CHANNELS,
                    rate=RATE,
                    input=True,
                    output=True,
                    stream_callback=callback)

    stream.start_stream()

    while stream.is_active():
        pass

# While the audio recording and vocalization process is done by the processor, the 2nd processor passes the raw audio data packets through the signal processing stages with 1024 samples added one after the other.
# First of all, by taking the absolute value of the raw audio data and optimizing the fluctuations in the signal, no data loss was experienced.
# I then removed background noise from the full wave rectified audio signal by equating values less than 0.3 to 0.
# By clearing the signal from background noise, I made it possible to distinguish between foreground sounds and got rid of unnecessary data.
# I got the amplitude envelope of the signal using the hilbert transform (scipy.signal.hilbert) which includes the Fourier transform.
# By removing the envelope of the signal, the high frequency signals were removed and the low frequency signals were retained.
# After going through the raw audio signal processing stages, I transferred it to the model by subsampling.
# In the sub-sampling process, 1024 samples were taken and the sound data, which was renewed and became continuous, was transferred to the model by downloading 116 samples.

def soundprocess(record2soundprocess, record2show, soundprocess2show, soundprocess2izi):
    old_voice = np.zeros(winSize * CHUNK)
    while True:
        if not record2soundprocess.empty():
            yeni_ses = record2soundprocess.get()

            old_voice = np.append(old_voice[-(winSize - 1) * CHUNK:], yeni_ses)
            old_voice = np.abs(old_voice)
            thr = 0.2
            i_clr = np.where(old_voice < thr)
            old_voice[i_clr] = 0;
            sound_hilbert = signal.hilbert(old_voice)
            envelope = np.abs(sound_hilbert)
            envelope = signal.convolve(envelope, np.ones(21), mode='same')

            record2show.put(old_voice)
            soundprocess2show.put(envelope)
            q = envelope[-CHUNK:]
            soundprocess2izi.put(signal.resample(q, 232))

def IZI(soundprocess2izi, izi2record):
    while True:
        while soundprocess2izi.empty():
            pass

        def Discrete_Model(a, b, u, v, I):
            v = v + h * (0.04 * v * v + 5 * v + 140 - u + I)
            u = u + h * (a * (b * v - u))
            return u, v

        def Izhikevich(a, b, c, d):
            v = -65 * np.ones(232)
            u = 0 * np.ones(232)
            u[0] = b * v[0]
            I = soundprocess2izi.get()

            for k in range(0, 116 - 1):
                u[k + 1], v[k + 1] = Discrete_Model(a, b, u[k], v[k], I[k])
                if v[k + 1] > 30:
                    u[k + 1] = u[k + 1] + d
                    v[k + 1] = c
            izi2record.put(signal.resample(v, 1024))

        #Izhikevich(0.02, 0.2, -55, 4)   # Intrinsically Bursting (IB)
        #Izhikevich(0.02, 0.2, -65, 8)  # Regular Spiking (RS)
        Izhikevich(0.1, 0.2, -65, 2)     # Fast Spiking (FS)

# In this code snippet, I introduced the Izhikevich neuron model.
# Mathematical equations are coded. The membrane potential variable and the membrane recovery variable were expressed in terms.
# The data making up the response was oversampled to generate an audio output.
# The code is completed by giving Fast Spiking parameters.

if __name__ == '__main__':
    record2soundprocess = Queue()
    record2show = Queue()
    soundprocess2izi = Queue()
    soundprocess2show = Queue()
    izi2record = Queue()

    fig, ax = plt.subplots(figsize=(15, 8))
    lineSes, = plt.plot(np.zeros(winSize * CHUNK), '-b')
    lineZarf, = plt.plot(np.zeros(winSize * CHUNK), '-r')

    ax.set_ylim(bottom=-1, top=1)

    p1 = Process(target=recordd, args=(record2soundprocess, izi2record))
    p2 = Process(target=soundprocess, args=(record2soundprocess, record2show, soundprocess2show, soundprocess2izi))
    p3 = Process(target=IZI, args=(soundprocess2izi, izi2record))
    p1.start()
    p2.start()
    p3.start()

    time.sleep(2)

    t0 = time.time()
    while (time.time() - t0) < 30:
        time.sleep(0.1)
        if not (record2show.empty() and soundprocess2show.empty()):
            ses = record2show.get()
            zarf = soundprocess2show.get()

            lineSes.set_ydata(ses)
            lineZarf.set_ydata(zarf)

            fig.canvas.draw()
            fig.canvas.flush_events()
    p1.terminate()
    p2.terminate()
    p3.terminate()
    time.sleep(1)

# In this part, which looks like the last part of the code but is actually the beginning, the data coming from the microphone is collected in one place and the processes are connected.
# Commands are given for visualization and 3 operations are defined.
# The first of these processes records the sound from the microphone and responds through the speaker, the other processes 1024 samples.
# The resulting processor generates the Izhikevich response.


from app import app
from app import server as ml
from flask import render_template, json, request, jsonify
import tweepy as tw
import csv
import pandas as pd
from werkzeug.utils import secure_filename
import os
from nltk.util import ngrams
import matplotlib.pyplot as plt

load = ml.LoadData()
collect = ml.Data_Scrape()
prepro = ml.Preprocessing()
net = ml.Network()
tm = ml.Timeseries()

RAW_DATA_LOC = 'app/static/file/collected/'
model_path = 'app/all_indo_man_tag_corpus_model.crf.tagger'
filter_path = RAW_DATA_LOC + 'Filter_tweet.csv'
ALLOWED_EXTENSIONS = {'csv', '','file'}

def set_default(obj):
    if isinstance(obj, set):
        return list(obj)
    raise TypeError

def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

def stopwords():
    if request.method == "POST":
        stopwords_df = load.loadData(filter_path)
        locHStopwords = load.savedata(RAW_DATA_LOC, stopwords_df['text'], "tweet_text.txt")
    return locHStopwords

def token():
    locHStopwords = stopwords()
    add_stopwords = request.form.get("stopwords")
    tweet_token = prepro.openandtoken(locHStopwords)
    tweet_clean_stopwords = prepro.stopwords_cleaner(tweet_token, add_stopwords)

    return tweet_clean_stopwords

@app.route("/twitter_data", methods=["GET", "POST"])
def gets_data():
    if request.method == "POST":
       search_words = request.form.get("search")
       date_since = request.form.get("since")
       date_until = request.form.get("until")
       numb_of_tweet = int(request.form.get("numb"))
       print(search_words)

       tweets = collect.get_data(search_words, date_since, date_until, numb_of_tweet)
       tweets_flat = prepro.flatten_tweets(tweets)
       ds_tweets = pd.DataFrame(tweets_flat)
       ds_tweets.drop(['entities','metadata','user','retweeted_status','source'], axis=1, inplace=True)
       print(ds_tweets)
       lochScrape = load.savedata(RAW_DATA_LOC, ds_tweets, "tweet2.csv")
       hasil = {
           "hScrape": {lochScrape}
           }
       res = json.dumps(hasil, default=set_default), 200
       return res
    return render_template("index.html")

@app.route("/filter_out", methods=["GET", "POST"])
def filter_data():
    if request.method == "POST":
        filter_words = []
        filter = request.form.get("filter-out")
        filter_words = [item for item in filter.split()]

        path = RAW_DATA_LOC + 'tweet.csv'
        filter_df = load.loadData(path)

        new_filter_df = filter_df[~filter_df.retweeted_status_screen_name.isin(filter_words)]
        print(new_filter_df)

        locHFilter = load.savedata(RAW_DATA_LOC, new_filter_df, "Filter_tweet.csv")
        hasil = {
           "hFilter": {locHFilter}
           }
        res = json.dumps(hasil, default=set_default), 200
        return res
    return render_template("filter.html")

@app.route("/exist_twitter", methods=["GET", "POST"])
def exist_twitter():
    if request.method == "POST":

        file = request.files["testDataset"]

        if file and allowed_file(file.filename):
            filename = secure_filename(file.filename)
            file.save(os.path.join(RAW_DATA_LOC, filename))
        else:
            return jsonify(message='error'), 500

        loc_test = os.path.join(RAW_DATA_LOC, filename)
        exist_df = load.loadData(loc_test)

        hasil = {"prediction": {loc_test}
                 }

        res = json.dumps(hasil, default=set_default), 200
        return res
    return render_template("index.html")

@app.route("/graph", methods=["GET", "POST"])
def graph():

    word1 = "#GarongKorupsiBerjamaah"
    word2 = "BPJS"

    ds_tweets = pd.read_csv('C:/Users/ASUS/nlp_analysis/app/static/file/collected/tweet.csv')
    ds_tweets = ds_tweets.sort_values(['created_at'], ascending=[True])
    ds_tweets['created_at'] = pd.to_datetime(ds_tweets['created_at'])
    ds_tweets = ds_tweets.set_index('created_at')

    timeseries_image = tm.get_timeseries_plot(ds_tweets, word1, word2)

    hasil = {
           "timeseries" : {timeseries_image}
           }
    res = json.dumps(hasil, default=set_default), 200
    return res

@app.route("/timeseries", methods=["GET", "POST"])
def timeseries_plot():
    if request.method == "POST":
        # word1 = request.form.get("word1")
        # word2 = request.form.get("word2")

        word1 = "#GarongKorupsiBerjamaah"
        word2 = "BPJS"

        ds_tweets = pd.read_csv('C:/Users/ASUS/nlp_analysis/app/static/file/collected/tweet.csv')
        ds_tweets = ds_tweets.sort_values(['created_at'], ascending=[True])
        ds_tweets['created_at'] = pd.to_datetime(ds_tweets['created_at'])
        ds_tweets = ds_tweets.set_index('created_at')

        timeseries_image = tm.get_timeseries_plot(ds_tweets, word1, word2)

        hasil = {
           "timeseries" : {timeseries_image}
           }
        res = json.dumps(hasil, default=set_default), 200
        return res
    return render_template("index.html")

@app.route("/stopwords", methods=["GET", "POST"])
def stopwords_custom():

    if request.method == "POST":

        unigram_threshold = 200
        bigram_threshold = 150
        trigram_threshold = 170
        verb_threshold = 30
        adjective_threshold = 10
        adverb_threshold = 2
        noun_threshold = 200

        locHStopwords = stopwords()

        tweet_clean_stopwords = token()

        wordcloud_image = prepro.wordcloud(tweet_clean_stopwords)
        unigrams_image = prepro.unigrams(tweet_clean_stopwords, unigram_threshold)
        bigrams_image = prepro.bigrams(tweet_clean_stopwords, bigram_threshold)
        trigrams_image = prepro.trigrams(tweet_clean_stopwords, trigram_threshold)

        result = prepro.pos_tagger(tweet_clean_stopwords, model_path)
        verbCounter = prepro.verb_counter(result, verb_threshold)
        adjectiveCounter = prepro.adjective_counter(result, adjective_threshold)
        adverbCounter = prepro.adverb_counter(result, adverb_threshold)
        nounCounter = prepro.noun_counter(result, noun_threshold)

        path = RAW_DATA_LOC + 'tweet.csv'
        ds_tweets = load.loadData(path)
        G_rt = net.RT_nodes_edges(ds_tweets)
        G_reply = net.reply_nodes_edges(ds_tweets)
        network_image = net.viz_network(G_rt)

        top_rt, top_reply = net.indegree_centrality(G_reply, G_rt)
        lochRT_indegree = load.savedata(RAW_DATA_LOC, top_rt, "RT_indegree.csv")
        lochReply_indegree = load.savedata(RAW_DATA_LOC, top_reply, "reply_indegree.csv")

        betweeness_rt, betweeness_reply = net.betweeness_centrality(G_rt, G_reply)
        lochRT_betweeness = load.savedata(RAW_DATA_LOC, betweeness_rt, "RT_betweeness.csv")
        lochReply_betweeness = load.savedata(RAW_DATA_LOC, betweeness_reply, "reply_betweeness.csv")

        ratio_df = net.ratio(G_rt, G_reply)
        lochRatio = load.savedata(RAW_DATA_LOC, ratio_df, "ratio.csv")

        hasil = {
           "hStopwords": {locHStopwords},
           "hRT_indegree": {lochRT_indegree},
            "hreply_indegree": {lochReply_indegree},
            "hRT_betweeness": {lochRT_betweeness},
            "hReply_betweeness": {lochReply_betweeness},
            "hRatio": {lochRatio},
           "wordcloud": {wordcloud_image},
           "network": {network_image},
           "unigram": {unigrams_image},
           "bigram": {bigrams_image},
           "trigram": {trigrams_image},
           "verb": {verbCounter},
           "adjective": {adjectiveCounter},
           "adverb": {adverbCounter},
           "noun": {nounCounter}
           }
        res = json.dumps(hasil, default=set_default), 200
        return res
    return render_template("index.html")

@app.route("/hUnigram", methods=["GET", "POST"])
def unigram():
    if request.method == "POST":
        tweet_clean_stopwords = token()
        unigram_threshold = int(request.form.get("unigram_threshold"))
        unigrams_image = prepro.unigrams(tweet_clean_stopwords, unigram_threshold)

        hasil = {
            "unigram": {unigrams_image}
            }
        res = json.dumps(hasil, default=set_default), 200
        return res
    return render_template("index.html")

@app.route("/hBigram", methods=["GET", "POST"])
def bigram():
    if request.method == "POST":
        tweet_clean_stopwords = token()
        bigram_threshold = int(request.form.get("bigram_threshold"))
        bigrams_image = prepro.bigrams(tweet_clean_stopwords, bigram_threshold)

        hasil = {
            "bigram": {bigrams_image}
            }
        res = json.dumps(hasil, default=set_default), 200
        return res
    return render_template("index.html")

@app.route("/hTrigram", methods=["GET", "POST"])
def trigram():
    if request.method == "POST":
        tweet_clean_stopwords = token()
        trigram_threshold = int(request.form.get("trigram_threshold"))
        trigrams_image = prepro.trigrams(tweet_clean_stopwords, trigram_threshold)

        hasil = {
            "trigram": {trigrams_image}
            }
        res = json.dumps(hasil, default=set_default), 200
        return res
    return render_template("index.html")

@app.route("/hVerb", methods=["GET", "POST"])
def verb():
    if request.method == "POST":
        tweet_clean_stopwords = token()
        result = prepro.pos_tagger(tweet_clean_stopwords, model_path)
        verb_threshold = int(request.form.get("verb_threshold"))
        verbCounter = prepro.verb_counter(result, verb_threshold)

        hasil = {
            "verb": {verbCounter}
            }
        res = json.dumps(hasil, default=set_default), 200
        return res
    return render_template("index.html")

@app.route("/hAdjective", methods=["GET", "POST"])
def adjective():
    if request.method == "POST":
        tweet_clean_stopwords = token()
        result = prepro.pos_tagger(tweet_clean_stopwords, model_path)
        adjective_threshold = int(request.form.get("adjective_threshold"))
        adjectiveCounter = prepro.adjective_counter(result, adjective_threshold)

        hasil = {
            "adjective": {adjectiveCounter}
            }
        res = json.dumps(hasil, default=set_default), 200
        return res
    return render_template("index.html")

@app.route("/hAdverb", methods=["GET", "POST"])
def adverb():
    if request.method == "POST":
        tweet_clean_stopwords = token()
        result = prepro.pos_tagger(tweet_clean_stopwords, model_path)
        adverb_threshold = int(request.form.get("adverb_threshold"))
        adverbCounter = prepro.adverb_counter(result, adverb_threshold)

        hasil = {
            "adverb": {adverbCounter}
            }
        res = json.dumps(hasil, default=set_default), 200
        return res
    return render_template("index.html")

@app.route("/hNoun", methods=["GET", "POST"])
def noun():
    if request.method == "POST":
        tweet_clean_stopwords = token()
        result = prepro.pos_tagger(tweet_clean_stopwords, model_path)
        noun_threshold = int(request.form.get("noun_threshold"))
        nounCounter = prepro.noun_counter(result, noun_threshold)

        hasil = {
            "noun": {nounCounter}
            }
        res = json.dumps(hasil, default=set_default), 200
        return res
    return render_template("index.html")

@app.route("/hScraping", methods=["GET"])
def tampil():
    data = []

    with open(RAW_DATA_LOC + 'tweet2.csv', encoding='utf-8') as csvfile:
        data_csv = csv.DictReader(csvfile)
        print(data_csv )
        for row in data_csv:
            data.append(dict(row))
    data = {"data": data}
    return jsonify(data)

@app.route("/hpResult", methods=["GET"])
def tampilResult():
    data = []

    with open(RAW_DATA_LOC + 'tweet.csv', encoding='utf-8') as csvfile:
        data_csv = csv.DictReader(csvfile, delimiter=',')
        for row in data_csv:
            data.append(dict(row))
    data = {"data": data}
    print(type(data))
    return jsonify(data)

@app.route("/hFilter", methods=["GET"])
def tampilFilter():
    data = []

    with open(RAW_DATA_LOC + 'Filter_tweet.csv', encoding='utf-8') as csvfile:
        data_csv = csv.DictReader(csvfile, delimiter=',')
        for row in data_csv:
            data.append(dict(row))
    data = {"data": data}
    return jsonify(data)

@app.route("/hRT_indegree", methods=["GET"])
def tampilRT_indgeree():
    data = []

    with open(RAW_DATA_LOC + 'RT_indegree.csv', encoding='utf-8') as csvfile:
        data_csv = csv.DictReader(csvfile, delimiter=',')
        for row in data_csv:
            data.append(dict(row))
    data = {"data": data}
    return jsonify(data)

@app.route("/hReply_indegree", methods=["GET"])
def tampilReply_indgeree():
    data = []

    with open(RAW_DATA_LOC + 'reply_indegree.csv', encoding='utf-8') as csvfile:
        data_csv = csv.DictReader(csvfile, delimiter=',')
        for row in data_csv:
            data.append(dict(row))
    data = {"data": data}
    return jsonify(data)

@app.route("/hRT_betweeness", methods=["GET"])
def tampilRT_betweeness():
    data = []

    with open(RAW_DATA_LOC + 'RT_betweeness.csv', encoding='utf-8') as csvfile:
        data_csv = csv.DictReader(csvfile, delimiter=',')
        for row in data_csv:
            data.append(dict(row))
    data = {"data": data}
    return jsonify(data)

@app.route("/hReply_betweeness", methods=["GET"])
def tampilReply_betweeness():
    data = []

    with open(RAW_DATA_LOC + 'reply_betweeness.csv', encoding='utf-8') as csvfile:
        data_csv = csv.DictReader(csvfile, delimiter=',')
        for row in data_csv:
            data.append(dict(row))
    data = {"data": data}
    return jsonify(data)

@app.route("/hRatio", methods=["GET"])
def tampilRatio():
    data = []

    with open(RAW_DATA_LOC + 'ratio.csv', encoding='utf-8') as csvfile:
        data_csv = csv.DictReader(csvfile, delimiter=',')
        for row in data_csv:
            data.append(dict(row))
    data = {"data": data}
    return jsonify(data)


"""
Creates custom Session objects for usage in the scraper,
compliant with the Wikimedia User-Agent policy, as given in
https://meta.wikimedia.org/wiki/User-Agent_policy

Refs:
https://docs.python-requests.org/en/master/user/advanced/#session-objects
https://toolbelt.readthedocs.io/en/latest/sessions.html#baseurlsession
https://findwork.dev/blog/advanced-usage-python-requests-timeouts-retries-hooks/
"""

import requests
from requests_toolbelt import sessions
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry

REQ_TIMEOUT = 15  # bad internet connection is bad
TIMES_TO_RETRY = 10

WIKI_ROOT = "https://en.wikipedia.org"
# no trailing / to make string concatenation simpler
# also no /wiki because that's included in scraped links

headers = {
    "User-Agent":
    "scraper-BOT-for-a-college-project/2.1"
    " (ferreira.mev@matematica.ufrj.br) "
    "python-requests/2.22.0",
    "Accept-Encoding": "gzip,deflate"
    }
    # the latter is as per
    # https://wikitech.wikimedia.org/wiki/Robot_policy

def raise_by_default(response, *args, **kwargs):
    response.raise_for_status()

class TimeoutHTTPAdapter(HTTPAdapter):
    """
    Custom HTTPAdapter subclass with a higher default timeout.
    """
    def __init__(self, *args, **kwargs):
        self.timeout = REQ_TIMEOUT
        super().__init__(*args, **kwargs)

    def send(self, request, **kwargs):
        if "timeout" not in kwargs:
            kwargs["timeout"] = self.timeout

        return super().send(request, **kwargs)

sesh = sessions.BaseUrlSession(base_url=WIKI_ROOT)

sesh.hooks["response"].append(raise_by_default)

retry_harder = Retry(
    total=TIMES_TO_RETRY,
    status_forcelist=[429, 500, 502, 503, 504],
    # Too Many Requests (guilty as charged) and some
    # server errors: generic Internal Server Error,
    # Bad Gateway, Service Unavailable, Gateway Timeout
    backoff_factor=1  # be kind to the servers :P
)

adapter = TimeoutHTTPAdapter(max_retries=retry_harder)
sesh.mount("https://", adapter)
sesh.mount("http://", adapter)


import os

import matplotlib.pyplot as plt
import numpy as np
import skill_metrics as sm

from aibedo.utilities.utils import get_logger
from aibedo.utilities.wandb_api import filter_wandb_runs, has_tags, get_predictions_xarray

log = get_logger(__name__)

tag = "MLP-all-4-constraints"
ref_dataset = 'era5'
save_dir = 'out_dir/taylor'
overrides = [
    '++datamodule.prediction_data=same_as_test',
    f'datamodule.partition=[0.85, 0.15, {ref_dataset}]',
    f'datamodule.num_workers={3}',
    'datamodule.eval_batch_size=10',
    'verbose=False',
    '++model.use_auxiliary_vars=False',
    # f'datamodule.data_dir={DATA_DIR}',
]
run_filters = [
    'has_finished',
    has_tags([tag]),
]
hyperparam_filter = None  # {'seed': 7}
runs = filter_wandb_runs(hyperparam_filter=hyperparam_filter, filter_functions=run_filters)
assert len(runs) > 0, "No runs found"
labels = dict()
corrs, stds, rmses = dict(), dict(), dict()

for i, run in enumerate(runs):
    if i > 3:
        pass  # continue
    run_id = run.id
    preds_xarray = get_predictions_xarray(
        run_id, overrides,
        also_errors=False, also_targets=True,
        split='predict',
        return_normalized_outputs=True,
        variables='all')
    assert preds_xarray.attrs['dataset_name'].lower() == ref_dataset.lower()
    esm_for_training = preds_xarray.attrs['esm_for_training']
    label_name = esm_for_training.upper()
    log.info(f" Evaluating {run_id} (trained on {esm_for_training})")

    out_vars = preds_xarray.attrs['variable_names'].split(';')
    for var in out_vars:
        if var in labels.keys() and label_name in labels[var]:
            continue
        ml_preds = preds_xarray[f'{var}_preds'].values
        ref = preds_xarray[f'{var}_targets'].values

        try:
            taylor_stats = sm.taylor_statistics(predicted=ml_preds.flatten(), reference=ref.flatten(), field='')
        except ValueError as e:
            log.warning(f"{esm_for_training} (ID={run_id}) raised exception on var {var}: {e}")
            break
        # Store statistics in arrays
        if i == 0:
            print(f"{var} stdev of ref:", taylor_stats['sdev'][0])
            stds[var] = [taylor_stats['sdev'][0], taylor_stats['sdev'][1]]
            rmses[var] = [taylor_stats['crmsd'][0], taylor_stats['crmsd'][1]]
            corrs[var] = [taylor_stats['ccoef'][0], taylor_stats['ccoef'][1]]
            labels[var] = [ref_dataset.upper()]
        else:
            assert np.isclose(stds[var][0], taylor_stats['sdev'][0])
            stds[var] += [taylor_stats['sdev'][1]]
            rmses[var] += [taylor_stats['crmsd'][1]]
            corrs[var] += [taylor_stats['ccoef'][1]]
        labels[var] += [label_name]

for var in out_vars:
    stds_var = np.array(stds[var])
    corrs_var = np.array(corrs[var])
    rmses_var = np.array(rmses[var])
    labels_var = labels[var]
    print(len(labels_var), stds_var.shape, corrs_var.shape, rmses_var.shape)
    sm.taylor_diagram(
        stds_var, rmses_var, corrs_var,
        checkstats=True,
        alpha=0.0,
        markersize=6,
        markerlegend='on',
        # markerdisplayed='colorbar',
        markerobs=True,
        markerLabel=labels_var,
        titleobs=ref_dataset.upper(),
        widthcor=0.5, widthrms=0.5, widthstd=0.6,
    )
    save_filename = os.path.join(save_dir, f"taylor_diagram_{ref_dataset.upper()}_{tag}_{var}.png")
    plt.savefig(save_filename, bbox_inches='tight')
    plt.close()


import torch
import torch.nn.functional as F
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as img
import torch.utils.data as data_utils
from unet_d3 import UNet  # depth of U-net change here
from torchsummary import summary
from loss import dice_loss, dice_coef, FGT_coe
from collections import defaultdict
import torch.optim as optim
from torch.optim import lr_scheduler
import os
import time
import copy
from skimage import color

TrainImgDir = '/home/jack/Documents/trainingdata_2classes_2_25/Train_Images/'
Train_class1Dir = '/home/jack/Documents/trainingdata_2classes_2_25/Class_1_FGT/'
Train_class2Dir = '/home/jack/Documents/trainingdata_2classes_2_25/Class_2_Breast_Whole/'

EvalImgDir = '/home/jack/Documents/trainingdata_2classes_2_25/Eval_Images/'
Eval_class1Dir = '/home/jack/Documents/trainingdata_2classes_2_25/Eval_Class_1/'
Eval_class2Dir = '/home/jack/Documents/trainingdata_2classes_2_25/Eval_Class_2/'
Eval_datasize = 10

Pred_Eval_class1Dir = '/home/jack/Documents/trainingdata_2classes_2_25/Prediction_Eval_Class_1/'
Pred_Eval_class2Dir = '/home/jack/Documents/trainingdata_2classes_2_25/Prediction_Eval_Class_2/'

TestImgDir = '/home/jack/Documents/trainingdata_2classes_2_25/Test_Images/'
Test_class1Dir = '/home/jack/Documents/trainingdata_2classes_2_25/Test_Class_1/'
Test_class2Dir = '/home/jack/Documents/trainingdata_2classes_2_25/Test_Class_2/'
Test_datasize = 30

Pred_Test_class1Dir = '/home/jack/Documents/trainingdata_2classes_2_25/Prediction_Test_Class_1/'
Pred_Test_class2Dir =  '/home/jack/Documents/trainingdata_2classes_2_25/Prediction_Test_Class_2/'

def importImg(TrainImgDir,Train_class1Dir,Train_class2Dir):
    Train_dataset_list = []
    Train_Class1_list = []
    Train_Class2_list = []

    for datafilename in sorted(os.listdir(TrainImgDir)):
        Train_dataset_list.append(datafilename)

    for labelfilename in sorted(os.listdir(Train_class1Dir)):
        Train_Class1_list.append(labelfilename)

    for labelfilename in sorted(os.listdir(Train_class2Dir)):
        Train_Class2_list.append(labelfilename)

    Train_dataset = torch.zeros(len(Train_dataset_list), 1, 256, 256)
    Train_Class1 = torch.zeros(len(Train_dataset_list), 256, 256)
    Train_Class2 = torch.zeros(len(Train_dataset_list), 256, 256)

    for file_index in range(len(Train_dataset_list)):
        Train_dataset[file_index, 0, :, :] = torch.tensor(img.imread(TrainImgDir + Train_dataset_list[file_index]))

        Train_label = img.imread(Train_class1Dir + Train_Class1_list[file_index])
        if Train_label.max() > 0:
            Train_label = Train_label/Train_label.max()
        Train_Class1[file_index, :, :] = torch.tensor(Train_label)

        Train_label = img.imread(Train_class2Dir + Train_Class2_list[file_index])[:, :, 0]
        if Train_label.max() > 0:
            Train_label = Train_label/Train_label.max()
        Train_Class2[file_index, :, :] = torch.tensor(Train_label)

    Train_labels = torch.empty(size=(len(Train_dataset_list), 2, 256, 256))
    Train_labels[:, 0, :, :] = Train_Class1
    Train_labels[:, 1, :, :] = Train_Class2
    return Train_dataset, Train_labels

def save_imgs(best_img,colormask_GT,Test_dataset,Pred_classDir_1,figIndex,tresh = 0.5):
    img.imsave(os.path.join(Pred_classDir_1 + 'best_' + str(figIndex) + '.tif'), best_img)
    best_img[best_img >= tresh] = 1
    best_img[best_img < tresh] = 0
    colormask_pred = best_img
    colormasks = np.zeros((256, 256, 3))
    mask_red = (colormask_pred == 1) & (colormask_GT == 0)
    mask_blue = (colormask_pred == 0) & (colormask_GT == 1)
    mask_yellow = (colormask_pred == 1) & (colormask_GT == 1)
    colormasks[mask_red] = [1,0,0]
    colormasks[mask_blue] = [0,0,1]
    colormasks[mask_yellow] = [1,1,0]
    BG_norm = Test_dataset[figIndex, 0, :, :] / Test_dataset.max()
    img_color = np.dstack((BG_norm, BG_norm, BG_norm))
    img_hsv = color.rgb2hsv(img_color)
    color_mask_hsv = color.rgb2hsv(colormasks)
    img_hsv[..., 0] = color_mask_hsv[..., 0]
    img_hsv[..., 1] = color_mask_hsv[..., 1] * 0.6  # alpha = 0.6
    img_masked = color.hsv2rgb(img_hsv)
    dice_coe = dice_coef(colormask_pred,colormask_GT)
    img.imsave(os.path.join(Pred_classDir_1 + 'best_overlay' + str(figIndex) +'_dice_'+ str("{:4f}".format(dice_coe))+'_.tif'), img_masked)

    return dice_coe

def calc_loss(pred, target, metrics, bce_weight=0.5):
    bce = F.binary_cross_entropy_with_logits(pred, target)
    pred = torch.sigmoid(pred)
    dice = dice_loss(pred, target)

    loss = bce * bce_weight + dice * (1 - bce_weight)

    metrics['bce'] += bce.data.cpu().numpy() * target.size(0)
    metrics['dice'] += dice.data.cpu().numpy() * target.size(0)
    metrics['loss'] += loss.data.cpu().numpy() * target.size(0)

    return loss

def print_metrics(metrics, epoch_samples, phase):
    outputs = []
    for k in metrics.keys():
        outputs.append("{}: {:4f}".format(k, metrics[k] / epoch_samples))

    print("{}: {}".format(phase, ", ".join(outputs)))

def train_model(model, optimizer, scheduler, num_epochs=25):
    best_model_wts = copy.deepcopy(model.state_dict())
    best_loss = 1e10

    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)

        since = time.time()

        # Each epoch has a training and validation phase
        for phase in ['train', 'val']:
            if phase == 'train':
                scheduler.step()
                for param_group in optimizer.param_groups:
                    print("LR", param_group['lr'])

                model.train()  # Set model to training mode
            else:
                model.eval()   # Set model to evaluate mode

            metrics = defaultdict(float)
            epoch_samples = 0

            for inputs, labels in dataloaders[phase]:
                inputs = inputs.cuda()
                labels = labels.cuda()

                # zero the parameter gradients
                optimizer.zero_grad()

                # forward
                # track history if only in train
                with torch.set_grad_enabled(phase == 'train'):
                    outputs = model(inputs)
                    loss = calc_loss(outputs, labels, metrics)

                    # backward + optimize only if in training phase
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()

                # statistics
                epoch_samples += inputs.size(0)

            if phase == 'val':
                pred_test = torch.sigmoid(outputs)
                pred_test = pred_test.data.cpu().numpy()
                for figIndex in range(6):
                    img.imsave(os.path.join(Pred_Eval_class1Dir +'epoch_'+str(epoch)+'_'+str(figIndex)+'.tif'), pred_test[figIndex,0,:,:])
                    img.imsave(os.path.join(Pred_Eval_class2Dir + 'epoch_' + str(epoch) + '_' + str(figIndex) + '.tif'), pred_test[figIndex, 1, :, :])

            print_metrics(metrics, epoch_samples, phase)
            epoch_loss = metrics['loss'] / epoch_samples

            # deep copy the model
            if phase == 'val' and epoch_loss < best_loss:
                print("saving best model")
                best_loss = epoch_loss
                best_model_wts = copy.deepcopy(model.state_dict())
                best_img = pred_test

        time_elapsed = time.time() - since
        print('{:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))

    print('Best val loss: {:4f}'.format(best_loss))
    dice_coe_class1 = 0
    dice_coe_class2 = 0
    for figIndex in range(Eval_datasize):
        dice_coe = save_imgs(best_img[figIndex, 0, :, :], Eval_labels[figIndex, 0, :, :].data.cpu().numpy(), Eval_dataset, Pred_Eval_class1Dir, figIndex)
        dice_coe_class1 += dice_coe
        dice_coe = save_imgs(best_img[figIndex, 1, :, :], Eval_labels[figIndex, 1, :, :].data.cpu().numpy(), Eval_dataset, Pred_Eval_class2Dir, figIndex)
        dice_coe_class2 += dice_coe
        print('Eval results: Predicted FGT% of figure {} : {:4f}'.format(figIndex,FGT_coe(best_img[figIndex, 0, :, :], best_img[figIndex,1,:,:])))
        print('Eval results: Real FGT% of figure {} : {:4f}'.format(figIndex,FGT_coe(Eval_labels[figIndex, 0, :, :].data.cpu().numpy(), Eval_labels[figIndex, 1, :, :].data.cpu().numpy())))
    print('Eval results: Best val class 1 dice coef: {:4f}, class 2 dice coef: {:4f}'.format(dice_coe_class1/Eval_datasize, dice_coe_class2/Eval_datasize))

    # load best model weights
    model.load_state_dict(best_model_wts)
    return model

# import training data, eval data, and test data from folders
Train_dataset, Train_labels = importImg(TrainImgDir,Train_class1Dir,Train_class2Dir)
Eval_dataset, Eval_labels = importImg(EvalImgDir,Eval_class1Dir,Eval_class2Dir)
Test_dataset, Test_labels = importImg(TestImgDir,Test_class1Dir,Test_class2Dir)

# load data into dataloader
train = data_utils.TensorDataset(Train_dataset, Train_labels)
Eval = data_utils.TensorDataset(Eval_dataset, Eval_labels)
dataloaders = { 'train' : data_utils.DataLoader(train, batch_size = 15, shuffle = True),
                'val'   : data_utils.DataLoader(Eval, batch_size = Eval_datasize, shuffle = False)
                }

# train loop for training data
# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # only for CPU or one PGU
model = UNet(n_class=2).cuda() # for 2 GPUs
gpus = [0,1]
model = torch.nn.DataParallel(model, device_ids=gpus)
summary(model, input_size=(1, 256, 256))
optimizer_ft = optim.Adam(model.parameters(),lr=1e-4)
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=30, gamma=0.1)
inputs, masks = next(iter(dataloaders['train']))
print(inputs.shape, masks.shape)
model = train_model(model, optimizer_ft, exp_lr_scheduler, num_epochs=100)

# test model
Test = data_utils.TensorDataset(Test_dataset, Test_labels)
Test_dataloader =  data_utils.DataLoader(Test, batch_size = Test_datasize, shuffle = False)
inputs, labels = next(iter(Test_dataloader))
pred = model(inputs)
pred_test = torch.sigmoid(pred)
pred_test = pred_test.data.cpu().numpy()
dice_coe_class1 = np.zeros(Test_datasize)
dice_coe_class2 = np.zeros(Test_datasize)
print('--- Test ---\n')
for figIndex in range(Test_datasize):
    dice_coe = save_imgs(pred_test[figIndex, 0, :, :], Test_labels[figIndex, 0, :, :].data.cpu().numpy(), Test_dataset, Pred_Test_class1Dir, figIndex)
    dice_coe_class1[figIndex]= dice_coe
    dice_coe = save_imgs(pred_test[figIndex, 1, :, :], Test_labels[figIndex, 1, :, :].data.cpu().numpy(), Test_dataset, Pred_Test_class2Dir, figIndex)
    dice_coe_class2[figIndex]= dice_coe
    print('Test results: Predicted FGT% of figure {} : {:4f}'.format(figIndex, FGT_coe(pred_test[figIndex, 0, :, :], pred_test[figIndex, 1, :, :])))
    print('Test results: Real FGT% of figure {} : {:4f}'.format(figIndex, FGT_coe(Test_labels[figIndex, 0, :, :].data.cpu().numpy(), Test_labels[figIndex, 1, :, :].data.cpu().numpy())))
print('Test results subject 1: mean class 1 dice coef: {:4f}, class 2 dice coef: {:4f} '.format(dice_coe_class1[0:9].sum()/10, dice_coe_class2[0:9].sum()/10))
print('Test results subject 2: mean class 1 dice coef: {:4f}, class 2 dice coef: {:4f} '.format(dice_coe_class1[10:19].sum()/10, dice_coe_class2[10:19].sum()/10))
print('Test results subject 3: mean class 1 dice coef: {:4f}, class 2 dice coef: {:4f} '.format(dice_coe_class1[20:29].sum()/10, dice_coe_class2[20:29].sum()/10))


from copy import deepcopy
from sklearn.metrics import classification_report, f1_score, precision_score
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import GaussianNB, MultinomialNB, BernoulliNB, ComplementNB
from sklearn import svm
import numpy as np
import pandas
import json

def take_the_best_genre(ct_dict):
    curr_max = 0
    curr_best = ""
    for key in ct_dict.keys():
        if ct_dict[key] > curr_max:
            curr_best = key
            curr_max = ct_dict[key]
    ct_dict.pop(curr_best)
    return curr_best

def json_str_to_one_genre(x):
    y = json.loads(x.strip())
    curr_index = 8
    curr_genre = ""
    for value in y.values():
            if value in chosen_genres and chosen_genres.index(value) < curr_index:
                curr_genre = value
                curr_index = chosen_genres.index(value)
    if curr_index != 8:
        return curr_genre
    else:
        return None

def f1score_mathched_elems(nb_engine, vectorizer, chunks_x, chunks_y):
    avg_prec = 0.0
    avg_f1_score = 0.0
    for chunk_idx in range(len(chunks_x)):
        curr_val_chunk_x = chunks_x[chunk_idx]
        curr_val_chunk_y = chunks_y[chunk_idx]
        tmp_chunks_x = list(chunks_x)
        tmp_chunks_y = list(chunks_y)
        tmp_chunks_x.pop(chunk_idx)
        tmp_chunks_y.pop(chunk_idx)

        curr_train_chunk_x = np.concatenate(tmp_chunks_x)
        curr_train_chunk_y = np.concatenate(tmp_chunks_y)
        y_pred = nb_engine.fit(vectorizer.transform(curr_train_chunk_x).toarray(), curr_train_chunk_y).predict(vectorizer.transform(curr_val_chunk_x).toarray())
        #print(curr_train_chunk_y)
        avg_prec += precision_score(curr_val_chunk_y, y_pred, average='weighted')
        avg_f1_score += f1_score(curr_val_chunk_y, y_pred, average='weighted')

    return (avg_prec / len(chunks_x), avg_f1_score / len(chunks_x))

data_to_process = pandas.read_csv('../booksummaries.txt', '\t', index_col=False, header=None)

data_to_process.drop(data_to_process.columns[[0, 1, 2, 3, 4]], axis=1, inplace=True)
data_to_process.dropna(inplace=True)

count_dict = {}
for column in data_to_process[[5]]:
    columnSeriesObj = data_to_process[column]
    for item in columnSeriesObj.values:
        y = json.loads(item.strip())
        for value in y.values():
            if count_dict.get(value):
                count_dict[value] = count_dict[value] + 1
            else:
                count_dict[value] = 1

#print(count_dict.values())
chosen_genres = []
for i in range(7):
    chosen_genres.append(take_the_best_genre(count_dict))

data_to_process[5] = data_to_process[5].transform(json_str_to_one_genre)

data_to_process.dropna(inplace=True)

cleansed_data = data_to_process.to_numpy()
train_n_validate_ds = cleansed_data[0:int(len(cleansed_data) * 0.9)]
test_ds = cleansed_data[int(len(cleansed_data) * 0.9):]
train_n_validate_ds_x = train_n_validate_ds[:,1]
vectorizer = TfidfVectorizer(stop_words='english', max_features=3000, decode_error='ignore')
vectorizer.fit(cleansed_data[:,1])
train_n_validate_ds_y = train_n_validate_ds[:,0]
test_ds_x = test_ds[:,1]
test_ds_y = test_ds[:,0]

train_n_validate_ds_x_chunks = np.array_split(train_n_validate_ds_x, 10)
train_n_validate_ds_y_chunks = np.array_split(train_n_validate_ds_y, 10)

# nb_engine1 = GaussianNB()
# print('Gausian Naive Bayes results: ')
# nb_eng_1_res = f1score_mathched_elems(nb_engine1, vectorizer, train_n_validate_ds_x_chunks, train_n_validate_ds_y_chunks)
# print(nb_eng_1_res)

# nb_engine2 = MultinomialNB()
# print('Multinomial Naive Bayes results: ')
# nb_eng_2_res = f1score_mathched_elems(nb_engine2, vectorizer, train_n_validate_ds_x_chunks, train_n_validate_ds_y_chunks)
# print(nb_eng_2_res)

# nb_engine3 = BernoulliNB()
# print('Bernoulli Naive Bayes results: ')
# nb_eng_3_res = f1score_mathched_elems(nb_engine3, vectorizer, train_n_validate_ds_x_chunks, train_n_validate_ds_y_chunks)
# print(nb_eng_3_res)

# nb_engine4 = ComplementNB()
# print('Complement Naive Bayes results: ')
# nb_eng_4_res = f1score_mathched_elems(nb_engine4, vectorizer, train_n_validate_ds_x_chunks, train_n_validate_ds_y_chunks)
# print(nb_eng_4_res)

# best_nb_engine = None
# if nb_eng_1_res[0] + nb_eng_1_res[1] * 1.25 > nb_eng_2_res[0] + nb_eng_2_res[1] * 1.25 and nb_eng_1_res[0] + nb_eng_1_res[1] * 1.25 > nb_eng_3_res[0] + nb_eng_3_res[1] * 1.25 and nb_eng_1_res[0] + nb_eng_1_res[1] * 1.25 > nb_eng_4_res[0] + nb_eng_4_res[1] * 1.25:
#     best_nb_engine = nb_engine1
# elif nb_eng_2_res[0] + nb_eng_2_res[1] * 1.25 > nb_eng_1_res[0] + nb_eng_1_res[1] * 1.25 and nb_eng_2_res[0] + nb_eng_2_res[1] * 1.25 > nb_eng_3_res[0] + nb_eng_3_res[1] * 1.25 and nb_eng_2_res[0] + nb_eng_2_res[1] * 1.25 > nb_eng_4_res[0] + nb_eng_4_res[1] * 1.25:
#     best_nb_engine = nb_engine2
# elif nb_eng_3_res[0] + nb_eng_3_res[1] * 1.25 > nb_eng_1_res[0] + nb_eng_1_res[1] * 1.25 and nb_eng_3_res[0] + nb_eng_3_res[1] * 1.25 > nb_eng_2_res[0] + nb_eng_2_res[1] * 1.25 and nb_eng_3_res[0] + nb_eng_3_res[1] * 1.25 > nb_eng_4_res[0] + nb_eng_4_res[1] * 1.25:
#     best_nb_engine = nb_engine3
# else:
#     best_nb_engine = nb_engine4

svm_engines = [svm.SVC(C=1.25, cache_size=325.0), svm.SVC(kernel='poly',degree=4, shrinking=False), svm.SVC(kernel='sigmoid', coef0=2.25)]

max_score = -1.0
best_svm_engine = svm_engines[0]
for engine in svm_engines:
    ret_tuple = f1score_mathched_elems(engine, vectorizer, train_n_validate_ds_x_chunks, train_n_validate_ds_y_chunks)
    print(ret_tuple)
    if ret_tuple[0] + ret_tuple[1] * 1.25 > max_score:
        max_score = ret_tuple[0] + ret_tuple[1] * 1.25
        best_nb_engine = engine

#best_nb_engine_score = f1score_mathched_elems(best_nb_engine, vectorizer, train_n_validate_ds_x_chunks, train_n_validate_ds_y_chunks)[0] + f1score_mathched_elems(best_nb_engine, vectorizer, train_n_validate_ds_x_chunks, train_n_validate_ds_y_chunks)[1] * 1.25
best_svm_engine_score = max_score
best_engine = best_svm_engine
# if best_nb_engine_score > best_svm_engine_score:
#     best_engine = best_nb_engine
# else:
#     best_engine = best_svm_engine

print('Predictions for the best classification engine')
y_pred = best_engine.predict(vectorizer.transform(test_ds_x).toarray())
print(classification_report(test_ds_y, y_pred))


from apscheduler.schedulers.blocking import BlockingScheduler
from datetime import datetime
import pushFCMNotification

def schedule_func():
    num = 1
    print(num)
    num += 1

    print(datetime.today())
    tokens = [pushFCMNotification.token_ella, pushFCMNotification.token_alpha]
    # for tok in tokens:
    #     pushFCMNotification.send_to_firebase_cloud_messaging(tok)
    pushFCMNotification.send_to_firebase_cloud_group_messaging(tokens)

if __name__ == "__main__":
    pushFCMNotification.init_app()

    scheduler = BlockingScheduler(timezone='Asia/Seoul', job_defaults={'max_instances': 1})
    scheduler.add_job(schedule_func, 'interval', minutes=1)

    # starting now
    for j in scheduler.get_jobs():
        j.modify(next_run_time=datetime.now())

    scheduler.start()


from setuptools import setup, find_packages

with open('requirements.txt', 'r') as f:
    requires = f.read().splitlines()

with open('README.md', 'r') as f:
    readme = f.read()

setup(name='hdmi-cec-to-adb',
      version='0.2',
      author='Paul Pavlish',
      author_email='hello@paulpavlish.com',
      url='https://github.com/paulsaccount/hdmi_cec_to_adb',
      description='hdmi_cec_to_adb',
      packages=find_packages(exclude=['*tests*']),
      long_description=readme,
      long_description_content_type='text/markdown',
      install_requires=requires,
      entry_points={
          'console_scripts': [
              'start_hdmi_cec_monitor = hdmi_cec_to_adb.bin.start_hdmi_cec_monitor:main',
          ],
      },
      )


import cv2
import numpy as np
import os

def verticalflip(img):
    origin = img.copy()
    verticalflip_img = cv2.flip(origin, 0)
    return verticalflip_img

def horizontalflip(img):
    origin = img.copy()
    horizontalflip_img = cv2.flip(origin, 1)
    return horizontalflip_img

def bothflip(img):
    origin = img.copy()
    bothflip_img = cv2.flip(origin, -1)
    return bothflip_img

def all_image_augmentation():
    image_list = [img for img in os.listdir('./eval')]
    image_list = sorted(image_list)
    print(image_list)
    for i in image_list:
        path = f'./eval/{i}'
        print(path)
        img = cv2.imread(path)
        verticalflip_img = verticalflip(img)
        horizontalflip_img = horizontalflip(img)
        bothflip_img = bothflip(img)
        cv2.imwrite(path[:20]+'-01.png', verticalflip_img)
        cv2.imwrite(path[:20]+'-02.png', horizontalflip_img)
        cv2.imwrite(path[:20]+'-03.png', bothflip_img)

if __name__ == '__main__':
    all_image_augmentation()


from enum import IntEnum
from threading import Lock

import models
# from piece import Piece

class InvalidBoardState(Exception):
    pass

class Direction(IntEnum):
    UP = 0
    DOWN = 1
    LEFT = 2
    RIGHT = 3

def get_neighbor(
    coord: models.Coordinate,
    direction: Direction,
    min_coord: int,
    max_coord: int,
) -> models.Coordinate:
    if coord is None:
        return None

    if direction == Direction.UP:
        return models.Coordinate(x=coord.x-1,y=coord.y) if coord.x > min_coord else None
    elif direction == Direction.DOWN:
        return models.Coordinate(x=coord.x+1,y=coord.y) if coord.x < max_coord - 1 else None
    elif direction == Direction.LEFT:
        return models.Coordinate(x=coord.x,y=coord.y-1) if coord.y > min_coord else None
    elif direction == Direction.RIGHT:
        return models.Coordinate(x=coord.x,y=coord.y+1) if coord.y < max_coord - 1 else None
    else:
        return None

class Board:
    # only ever one Board object with consistent board state

    def __init__(self, dimension):
        self.dimension = dimension
        self.board = [[None]*dimension for _ in range(dimension)]
        self.lock = Lock()

    def __str__(self):
        s = "[" + "---"*self.dimension + "]\n"
        for i, _ in enumerate(self.board):
            s += "["
            for j, _ in enumerate(self.board):
                s += "   " if not self.board[i][j] else f" {self.board[i][j]} "
            s += "]\n"
        s += "[" + "---"*self.dimension + "]"
        return s

    def __enter__(self):
        try:
            with self.lock:
                return self
        except:
            pass

    def __exit__(self, *_exc):
        pass

    def place(
        self,
        piece: models.Piece,
        origin: models.Coordinate,
        player_id: int
    ):
        print(origin, piece)
        valid_corner = False
        for coord in piece.shape:
            abs_coord = models.Coordinate(x=origin.x+coord.x, y=origin.y+coord.y)
            if self.get_board_square(abs_coord) is not None:
                raise InvalidBoardState

            if self.has_self_side(abs_coord, player_id, max_coord=self.dimension):
                raise InvalidBoardState

            valid_corner = (
                valid_corner or
                coord == models.Coordinate(x=0,y=0) or
                self.has_valid_corner(abs_coord, player_id, max_coord=self.dimension)
            )

        if not valid_corner:
            raise InvalidBoardState

        # All checks pass, this is a valid board state
        for coord in piece.shape:
            abs_coord = models.Coordinate(x=origin.x+coord.x, y=origin.y+coord.y)
            self.board[abs_coord.x][abs_coord.y] = player_id

        print(self)

    def get_board_square(self, coord: models.Coordinate) -> int:
        print(coord.x, coord.y)
        return self.board[coord.x][coord.y]

    def occupied_by_player(self, pid: int, coord: models.Coordinate) -> bool:
        return self.get_board_square(coord) == pid

    def has_self_side(
        self,
        coord: models.Coordinate,
        pid: int,
        max_coord: int,
        min_coord: int = 0,
    ):

        def get_neighbor_helper(dir: Direction) -> models.Coordinate:
            return get_neighbor(coord, dir, min_coord, max_coord)

        def occupied_by_pid(coord: models.Coordinate) -> bool:
            return self.occupied_by_player(pid, coord)

        l = get_neighbor_helper(Direction.LEFT)
        r = get_neighbor_helper(Direction.RIGHT)
        u = get_neighbor_helper(Direction.UP)
        d = get_neighbor_helper(Direction.DOWN)
        return (
            (l is not None and occupied_by_pid(l)) or
            (r is not None and occupied_by_pid(r)) or
            (u is not None and occupied_by_pid(u)) or
            (d is not None and occupied_by_pid(d))
        )

    def has_valid_corner(
        self,
        coord: models.Coordinate,
        pid: int,
        max_coord: int,
        min_coord: int = 0,
    ):
        def get_neighbor_helper(coord, dir: Direction) -> models.Coordinate:
            return get_neighbor(coord, dir, min_coord, max_coord)

        def occupied_by_pid(coord: models.Coordinate) -> bool:
            return self.occupied_by_player(pid, coord)

        ul = get_neighbor_helper(get_neighbor_helper(coord, Direction.UP), Direction.LEFT)
        ur = get_neighbor_helper(get_neighbor_helper(coord, Direction.UP), Direction.RIGHT)
        dl = get_neighbor_helper(get_neighbor_helper(coord, Direction.DOWN), Direction.LEFT)
        dr = get_neighbor_helper(get_neighbor_helper(coord, Direction.DOWN), Direction.RIGHT)
        return (
            (ul is not None and occupied_by_pid(ul)) or
            (ur is not None and occupied_by_pid(ur)) or
            (dl is not None and occupied_by_pid(dl)) or
            (dr is not None and occupied_by_pid(dr))
        )


from kmeans import KMeansClustering
from dataset import Dataset
from datetime import timedelta, date, datetime
from functools import reduce
import plotly.graph_objects as go
import pandas as pd
import pickle
import numpy as np
import os
import json

class ManagementModel:
    def __init__(self):
        self.dataset = Dataset()
        self.clustering_model = KMeansClustering()
        self.config = None
        self.name_goal = None

    def set_config(self, config):#, data):
        self.config = config

    def set_dataset(self, data):
        # formatte les données pour les envoyer au modèle
        # récupère la configuration du format pour la sauvegarder
        self.config = self.dataset.set(self.config, data)

    def set_model(self):
        self.clustering_model.set(self.config)

    def train(self):
        data = self.dataset.data.copy()
        data = self.dataset.set_data(data)
        list_format_column = self.config['list_format_column']
        index_vibration = list_format_column.index("vibration")

        if 'timestamp' in list_format_column:
            index_goal = list_format_column.index("timestamp")
            formats = ['%Y-%m-%d %H:%M:%S.%f', '%d/%m/%Y %H:%M','%m/%d/%Y  %H:%M:%S', '%d-%m-%y', '%d-%m-%Y', '%Y-%m-%d', '%b %y', '%B %Y', '%d %B %Y']
            data['timestamp'] = reduce(lambda l, r: l.combine_first(r), [pd.to_datetime(data['timestamp'], format=fmt, errors='coerce') for fmt in formats])

        self.clustering_model.train(self.config, data[['vibration']])

    def demo(self, data_in):

        data = data_in.copy()

        data = data.reset_index(drop=True)

        list_format_column = self.config['list_format_column']
        index_vibration = list_format_column.index("vibration")
        if 'timestamp' in list_format_column:
            index_goal = list_format_column.index("timestamp")
            formats = ['%Y-%m-%d %H:%M:%S.%f', '%d/%m/%Y %H:%M','%m/%d/%Y  %H:%M:%S', '%d-%m-%y', '%d-%m-%Y', '%Y-%m-%d', '%b %y', '%B %Y', '%d %B %Y']
            data.iloc[:, index_goal] = reduce(lambda l, r: l.combine_first(r), [pd.to_datetime(data.iloc[:, index_goal], format=fmt, errors='coerce') for fmt in formats])

            last_date_str = data.iloc[[-1],index_goal].values[0]
            last_date = last_date_str

            liste_date_all = []
            for index, row in data.iterrows():
                dat_str = row[index_goal]
                date  = dat_str
                liste_date_all.append(date)

        else:
            liste_date_all = list(range(len(data)))

        data = self.dataset.set_data(data)
        fig = go.Figure()
        df_results = self.clustering_model.fit_predict(data[['vibration']])
        dico_cst = self.min_max_mean(df_results)
        fig.add_trace(go.Scatter(x=liste_date_all, y=data['vibration'], mode = 'markers', marker = dict(color = df_results['cluster'], colorscale = 'Viridis')))
        fig.add_hline(y=dico_cst['max'], line_color='red')
        fig.add_hline(y=dico_cst['min'], line_color='red')
        fig.add_hline(y=dico_cst['low_mean'], line_color='green')
        fig.add_hline(y=dico_cst['high_mean'], line_color='green')
        fig.show()
        return data

    def min_max_mean(self, data):
        dico = {}
        clusters = data['cluster'].value_counts().copy()
        clusters = clusters.sort_values(ascending=False)
        clusters = clusters.reset_index()
        clusters['cluster'].sum()
        index = 0
        nb_points = 0
        for i in range(len(clusters)):
            if(nb_points < clusters['cluster'].sum()*0.75):
                nb_points += clusters.loc[i, 'cluster']
        else:
            index = clusters.loc[i, 'index']
        cluster1 = clusters.loc[index-1, :]
        cluster2 = clusters.loc[index, :]

        mean_cluster1 = data[data['cluster'] == cluster1['index']]['vibration'].mean()
        mean_cluster2 = data[data['cluster'] == cluster2['index']]['vibration'].mean()

        if (mean_cluster1 > mean_cluster2):
            max_vib = data[data['cluster'] == cluster1['index']]['vibration'].max()
            min_vib = data[data['cluster'] == cluster2['index']]['vibration'].min()
            dico['low_mean'] = mean_cluster2
            dico['high_mean'] = mean_cluster1
            dico['max'] = max_vib
            dico['min'] = min_vib
        else:
            max_vib = data[data['cluster'] == cluster2['index']]['vibration'].max()
            min_vib = data[data['cluster'] == cluster1['index']]['vibration'].min()
            dico['low_mean'] = mean_cluster1
            dico['high_mean'] = mean_cluster2
            dico['max'] = max_vib
            dico['min'] = min_vib
        return dico

    def calcul(self, data):
        x = self.dataset.create_input(data)
        return True

    def save(self):
        # Sauvegarde des configurations du modèle
        if self.clustering_model.model_is_trained:
            name_file = self.config['n_clusters']
            with open(f'model/{name_file}.pkl', 'wb') as file:
                pickle.dump(self.clustering_model, file)
        else:
            print("Aucun modèle à sauvegarder.")

    def load(self, config, data, model):
        del self.dataset
        del self.clustering_model

        self.dataset = Dataset()
        self.clustering_model = KMeansClustering()

        print("Chargement du model")

        self.set_config(config, data)
        self.set_dataset(data)

        self.clustering_model.load(config, model)

        return True


import math
import scipy.optimize as optimize
import constants as const
import conversion_factors as conver

class FanningMethod(object):

    def __init__(self, well):
        self.well = well
        self.length = None
        self.pressure_output = None
        self.temperature_average = None

    def friction_factor_fanning(self, pressure, temperature):
        """Расчет коэффициента трения для сегмента трубы по Fanning.

        Note:
            Источник: http://fekete.com/SAN/TheoryAndEquations/PiperTheoryEquations/c-te-pressure.htm.

        Args:
            pressure (float): Давление флюида, barsa.
            temperature (float): Температура флюида, K.

        Returns:
            friction_factor_fanning (float): Коэффициент трения для сегмента трубы с флюидом по Fanning.

        """
        well = self.well
        pipe_production = well.pipe_production
        roughness_absolute = pipe_production.roughness_absolute
        diameter_inner = pipe_production.diameter_inner
        roughness_relative = roughness_absolute / diameter_inner
        number_reynolds = well.number_reynolds(pressure, temperature)
        term1 = 0.2698 * roughness_relative
        term2 = 5.0452 / number_reynolds
        term3 = math.log(0.3539 * roughness_relative ** 1.1098 + 5.8506 / number_reynolds ** 0.8981)
        friction_factor_fanning = 1 / (-4 * math.log(term1 - term2 * term3)) ** 2
        return friction_factor_fanning

    def pressure_difference_hydrostatic(self, length_pipe_segment, pressure, temperature):
        """Расчет изменения гидростатического давления в сегменте трубы.

        Note:
            Источник: http://fekete.com/SAN/TheoryAndEquations/PiperTheoryEquations/c-te-pressure.htm.

        Args:
            length_pipe_segment (float): Длина сегмента трубы, m.
            pressure (float): Среднее давление флюида по длине сегмента, barsa.
            temperature (float): Средняя температура флюида по длине сегмента, K.

        Returns:
            pressure_loss_hydrostatic (float): Изменение гидростатического давления газа в сегменте трубы, barsa.

        """
        constant_gravity = const.CONSTANT_GRAVITY
        constant_gravity *= (conver.bar_to_Pa * conver.m_to_ft)
        well = self.well
        gas = well.fluid.phases[0]
        density = gas.density(pressure, temperature)
        density *= (conver.kg_to_lb / conver.m_to_ft ** 3)
        length_pipe_segment *= conver.m_to_ft
        pressure_loss_hydrostatic = constant_gravity / (144 * constant_gravity) * density * length_pipe_segment
        pressure_loss_hydrostatic *= conver.psi_to_bar
        return pressure_loss_hydrostatic

    def pressure_difference_friction(self, length_pipe_segment, pressure, temperature):
        """Расчет потери давления на трение в сегменте трубы.

        Note:
            Используется корреляция Fanning.
            Источник: http://fekete.com/SAN/TheoryAndEquations/PiperTheoryEquations/c-te-pressure.htm.

        Args:
            length_pipe_segment (float): Длина сегмента трубы, m.
            pressure (float): Среднее давление флюида по длине сегмента, barsa.
            temperature (float): Средняя температура флюида по длине сегмента, K.

        Returns:
            __calc_pressure_difference_friction (float): Потеря давления флюида на трение в сегменте трубы, barsa.

        """
        friction_factor_fanning = self.friction_factor_fanning(pressure, temperature)
        well = self.well
        gas = well.fluid.phases[0]
        density = gas.density(pressure, temperature)
        density_standard = gas.density(const.PRESSURE_STANDARD, const.TEMPERATURE_STANDARD)
        rate_standard = well.fluid.rates_standard[gas]
        rate = density_standard * rate_standard / density
        pipe_production = well.pipe_production
        diameter_inner = pipe_production.diameter_inner
        area_flow = math.pi * diameter_inner ** 2 / 4
        velocity = rate / area_flow

        density *= (conver.kg_to_lb / conver.m_to_ft ** 3)
        velocity *= (conver.m_to_ft / conver.day_to_s)
        length_pipe_segment *= conver.m_to_ft
        constant_gravity = const.CONSTANT_GRAVITY
        constant_gravity *= (conver.bar_to_Pa * conver.m_to_ft)
        diameter_inner *= conver.m_to_ft

        term1 = 2 * friction_factor_fanning * density * velocity ** 2 * length_pipe_segment
        term2 = constant_gravity * diameter_inner
        pressure_loss_friction = term1 / term2
        pressure_loss_friction *= conver.psi_to_bar
        return pressure_loss_friction

    def target_function(self, pressure_input):
        pressure_output = self.pressure_output
        pressure_average = (pressure_output + pressure_input) / 2
        length = self.length
        temperature_average = self.temperature_average
        pressure_difference_hydrostatic = self.pressure_difference_hydrostatic(length, pressure_average, temperature_average)
        pressure_loss_friction = self.pressure_difference_friction(length, pressure_average, temperature_average)
        _pressure_input = pressure_output + (pressure_difference_hydrostatic + pressure_loss_friction)
        error_pressure_input = pressure_input - _pressure_input
        return error_pressure_input

    def compute(self, length, pressure_output, temperature_average):
        self.length = length
        self.pressure_output = pressure_output
        self.temperature_average = temperature_average
        target_function = self.target_function
        pressure_input = optimize.root_scalar(target_function, method='brenth', bracket=[pressure_output, 1e3]).root
        return pressure_input


#!/usr/bin/env python3

import json

import requests

SLACK_API_KEY = "get token from https://api.slack.com/custom-integrations/legacy-tokens"
OPENWEATHER_API_KEY = "get token from https://openweathermap.org/appid"
OPENWEATHER_CITY_ID = "find your city id https://openweathermap.org/current#cityid"

# https://openweathermap.org/weather-conditions
WEATHER_CODES = {
    '2': ':lightning:',
    '3': ':rain_cloud:',
    '5': ':rain_cloud:',
    '6': ':snowflake:',
    '7': ':fog:',
    '80': ':cloud:',
    '800': ':sunny:',
    '801': ':sun_small_cloud:',
}

def get_emoji(code):
    code = str(code)
    if code in WEATHER_CODES:
        return WEATHER_CODES.get(code)

    if code[:2] in WEATHER_CODES:
        return WEATHER_CODES.get(code[:2])

    if code[:1] in WEATHER_CODES:
        return WEATHER_CODES.get(code[:1])

def to_celsius(kelvin):
    return kelvin - 273.15

def build_status(text, emoji):
    payload = {
        'status_text': text,
        'status_emoji': emoji
    }
    return json.dumps(payload)

if __name__ == '__main__':
    r = requests.get('http://api.openweathermap.org/data/2.5/weather?id={}&appid={}'.format(OPENWEATHER_CITY_ID,
                                                                                OPENWEATHER_API_KEY))

    weather = r.json().get('weather')[0]
    temp = to_celsius(r.json().get('main').get('temp'))

    text = '{}°C and {}'.format(round(temp), weather.get('description'))
    emoji = get_emoji(weather.get('id'))

    r = requests.post('https://slack.com/api/users.profile.set', data={
        'token': SLACK_API_KEY,
        'profile': build_status(text, emoji)
    })

    if r.status_code != 200:
        print(r.status_code)
        print(r.json())


from django.shortcuts import render, redirect
from .models import Article
from .forms import ArticleForm

# Create your views here.

# 요청 정보를 받아서..
def index(request):
    # DB에서 가져와서
    articles = Article.objects.order_by('-pk')
    # Template에 전달한다
    context = {
        'articles': articles
    }
    return render(request, 'articles/index.html', context)

# def new(request):
#     article_form = ArticleForm()
#     context = {
#         'article_form': article_form
#     }
#     return render(request, 'articles/new.html', context=context)

# new와 create 함수 합치기
def create(request):
    # form에서 글쓰기를 누르면 실행
    if request.method == 'POST':
        # 입력받은 데이터들 DB에 저장
        article_form = ArticleForm(request.POST)
        # 유효할 경우 실행 (valid)
        if article_form.is_valid():
            article_form.save()
            # index 페이지로 redirect
            return redirect('articles:index')
    # 'GET' method 일때
    # http://127.0.0.1:8000/articles/create 일때 실행
    else:
        article_form = ArticleForm()
    # 유효하지 않을 경우 아래로 이동 (invalid)
    context = {
        'article_form': article_form
    }
    return render(request, 'articles/new.html', context=context)

def detail(request, pk):
    # 특정 글을 가져온다
    article = Article.objects.get(pk=pk)
    # template에 객체 전달
    context = {
        'article': article
    }
    return render(request, 'articles/detail.html', context)