import os
import os.path as osp
import pickle
from typing import Optional

import numpy as np
import torch
import torch.nn as nn

from .body_models import SMPL
from .lbs import find_dynamic_lmk_idx_and_bcoords, lbs, vertices2landmarks
from .utils import (FLAMEOutput, Struct, Tensor, find_joint_kin_chain, to_np,
                    to_tensor)

class FLAME(SMPL):
    NUM_JOINTS = 5
    SHAPE_SPACE_DIM = 300
    EXPRESSION_SPACE_DIM = 100
    NECK_IDX = 0

    def __init__(
        self,
        model_path: str,
        data_struct=None,
        num_expression_coeffs=10,
        create_expression: bool = True,
        expression: Optional[Tensor] = None,
        create_neck_pose: bool = True,
        neck_pose: Optional[Tensor] = None,
        create_jaw_pose: bool = True,
        jaw_pose: Optional[Tensor] = None,
        create_leye_pose: bool = True,
        leye_pose: Optional[Tensor] = None,
        create_reye_pose=True,
        reye_pose: Optional[Tensor] = None,
        use_face_contour=False,
        batch_size: int = 1,
        gender: str = 'neutral',
        dtype: torch.dtype = torch.float32,
        ext='pkl',
        **kwargs
    ) -> None:
        ''' FLAME model constructor
            Parameters
            ----------
            model_path: str
                The path to the folder or to the file where the model
                parameters are stored
            num_expression_coeffs: int, optional
                Number of expression components to use
                (default = 10).
            create_expression: bool, optional
                Flag for creating a member variable for the expression space
                (default = True).
            expression: torch.tensor, optional, Bx10
                The default value for the expression member variable.
                (default = None)
            create_neck_pose: bool, optional
                Flag for creating a member variable for the neck pose.
                (default = False)
            neck_pose: torch.tensor, optional, Bx3
                The default value for the neck pose variable.
                (default = None)
            create_jaw_pose: bool, optional
                Flag for creating a member variable for the jaw pose.
                (default = False)
            jaw_pose: torch.tensor, optional, Bx3
                The default value for the jaw pose variable.
                (default = None)
            create_leye_pose: bool, optional
                Flag for creating a member variable for the left eye pose.
                (default = False)
            leye_pose: torch.tensor, optional, Bx10
                The default value for the left eye pose variable.
                (default = None)
            create_reye_pose: bool, optional
                Flag for creating a member variable for the right eye pose.
                (default = False)
            reye_pose: torch.tensor, optional, Bx10
                The default value for the right eye pose variable.
                (default = None)
            use_face_contour: bool, optional
                Whether to compute the keypoints that form the facial contour
            batch_size: int, optional
                The batch size used for creating the member variables
            gender: str, optional
                Which gender to load
            dtype: torch.dtype
                The data type for the created variables
        '''
        model_fn = f'FLAME_{gender.upper()}.{ext}'
        flame_path = os.path.join(model_path, model_fn)
        assert osp.exists(flame_path), 'Path {} does not exist!'.format(
            flame_path)
        if ext == 'npz':
            file_data = np.load(flame_path, allow_pickle=True)
        elif ext == 'pkl':
            with open(flame_path, 'rb') as smpl_file:
                file_data = pickle.load(smpl_file, encoding='latin1')
        else:
            raise ValueError('Unknown extension: {}'.format(ext))
        data_struct = Struct(**file_data)

        super(FLAME, self).__init__(
            model_path=model_path,
            data_struct=data_struct,
            dtype=dtype,
            batch_size=batch_size,
            gender=gender,
            ext=ext,
            **kwargs)

        self.use_face_contour = use_face_contour

        self.vertex_joint_selector.extra_joints_idxs = to_tensor(
            [], dtype=torch.long)

        if create_neck_pose:
            if neck_pose is None:
                default_neck_pose = torch.zeros([batch_size, 3], dtype=dtype)
            else:
                default_neck_pose = torch.tensor(neck_pose, dtype=dtype)
            neck_pose_param = nn.Parameter(
                default_neck_pose, requires_grad=True)
            self.register_parameter('neck_pose', neck_pose_param)

        if create_jaw_pose:
            if jaw_pose is None:
                default_jaw_pose = torch.zeros([batch_size, 3], dtype=dtype)
            else:
                default_jaw_pose = torch.tensor(jaw_pose, dtype=dtype)
            jaw_pose_param = nn.Parameter(default_jaw_pose,
                                          requires_grad=True)
            self.register_parameter('jaw_pose', jaw_pose_param)

        if create_leye_pose:
            if leye_pose is None:
                default_leye_pose = torch.zeros([batch_size, 3], dtype=dtype)
            else:
                default_leye_pose = torch.tensor(leye_pose, dtype=dtype)
            leye_pose_param = nn.Parameter(default_leye_pose,
                                           requires_grad=True)
            self.register_parameter('leye_pose', leye_pose_param)

        if create_reye_pose:
            if reye_pose is None:
                default_reye_pose = torch.zeros([batch_size, 3], dtype=dtype)
            else:
                default_reye_pose = torch.tensor(reye_pose, dtype=dtype)
            reye_pose_param = nn.Parameter(default_reye_pose,
                                           requires_grad=True)
            self.register_parameter('reye_pose', reye_pose_param)

        shapedirs = data_struct.shapedirs
        if len(shapedirs.shape) < 3:
            shapedirs = shapedirs[:, :, None]
        if (shapedirs.shape[-1] < self.SHAPE_SPACE_DIM +
                self.EXPRESSION_SPACE_DIM):
            print(f'WARNING: You are using a {self.name()} model, with only'
                  ' 10 shape and 10 expression coefficients.')
            expr_start_idx = 10
            expr_end_idx = 20
            num_expression_coeffs = min(num_expression_coeffs, 10)
        else:
            expr_start_idx = self.SHAPE_SPACE_DIM
            expr_end_idx = self.SHAPE_SPACE_DIM + num_expression_coeffs
            num_expression_coeffs = min(
                num_expression_coeffs, self.EXPRESSION_SPACE_DIM)

        self._num_expression_coeffs = num_expression_coeffs

        expr_dirs = shapedirs[:, :, expr_start_idx:expr_end_idx]
        self.register_buffer(
            'expr_dirs', to_tensor(to_np(expr_dirs), dtype=dtype))

        if create_expression:
            if expression is None:
                default_expression = torch.zeros(
                    [batch_size, self.num_expression_coeffs], dtype=dtype)
            else:
                default_expression = torch.tensor(expression, dtype=dtype)
            expression_param = nn.Parameter(default_expression,
                                            requires_grad=True)
            self.register_parameter('expression', expression_param)

        # The pickle file that contains the barycentric coordinates for
        # regressing the landmarks
        landmark_bcoord_filename = osp.join(
            model_path, 'flame_static_embedding.pkl')

        with open(landmark_bcoord_filename, 'rb') as fp:
            landmarks_data = pickle.load(fp, encoding='latin1')

        lmk_faces_idx = landmarks_data['lmk_face_idx'].astype(np.int64)
        self.register_buffer('lmk_faces_idx',
                             torch.tensor(lmk_faces_idx, dtype=torch.long))
        lmk_bary_coords = landmarks_data['lmk_b_coords']
        self.register_buffer('lmk_bary_coords',
                             torch.tensor(lmk_bary_coords, dtype=dtype))
        if self.use_face_contour:
            face_contour_path = os.path.join(
                model_path, 'flame_dynamic_embedding.npy')
            contour_embeddings = np.load(face_contour_path,
                                         allow_pickle=True,
                                         encoding='latin1')[()]

            dynamic_lmk_faces_idx = np.array(
                contour_embeddings['lmk_face_idx'], dtype=np.int64)
            dynamic_lmk_faces_idx = torch.tensor(
                dynamic_lmk_faces_idx,
                dtype=torch.long)
            self.register_buffer('dynamic_lmk_faces_idx',
                                 dynamic_lmk_faces_idx)

            dynamic_lmk_b_coords = torch.tensor(
                contour_embeddings['lmk_b_coords'], dtype=dtype)
            self.register_buffer(
                'dynamic_lmk_bary_coords', dynamic_lmk_b_coords)

            neck_kin_chain = find_joint_kin_chain(self.NECK_IDX, self.parents)
            self.register_buffer(
                'neck_kin_chain',
                torch.tensor(neck_kin_chain, dtype=torch.long))

    @property
    def num_expression_coeffs(self):
        return self._num_expression_coeffs

    def name(self) -> str:
        return 'FLAME'

    def extra_repr(self):
        msg = [
            super(FLAME, self).extra_repr(),
            f'Number of Expression Coefficients: {self.num_expression_coeffs}',
            f'Use face contour: {self.use_face_contour}',
        ]
        return '\n'.join(msg)

    def forward(
        self,
        betas: Optional[Tensor] = None,
        global_orient: Optional[Tensor] = None,
        neck_pose: Optional[Tensor] = None,
        transl: Optional[Tensor] = None,
        expression: Optional[Tensor] = None,
        jaw_pose: Optional[Tensor] = None,
        leye_pose: Optional[Tensor] = None,
        reye_pose: Optional[Tensor] = None,
        return_verts: bool = True,
        return_full_pose: bool = False,
        pose2rot: bool = True,
        **kwargs
    ) -> FLAMEOutput:
        '''
        Forward pass for the SMPLX model
            Parameters
            ----------
            global_orient: torch.tensor, optional, shape Bx3
                If given, ignore the member variable and use it as the global
                rotation of the body. Useful if someone wishes to predicts this
                with an external model. (default=None)
            betas: torch.tensor, optional, shape Bx10
                If given, ignore the member variable `betas` and use it
                instead. For example, it can used if shape parameters
                `betas` are predicted from some external model.
                (default=None)
            expression: torch.tensor, optional, shape Bx10
                If given, ignore the member variable `expression` and use it
                instead. For example, it can used if expression parameters
                `expression` are predicted from some external model.
            jaw_pose: torch.tensor, optional, shape Bx3
                If given, ignore the member variable `jaw_pose` and
                use this instead. It should either joint rotations in
                axis-angle format.
            jaw_pose: torch.tensor, optional, shape Bx3
                If given, ignore the member variable `jaw_pose` and
                use this instead. It should either joint rotations in
                axis-angle format.
            transl: torch.tensor, optional, shape Bx3
                If given, ignore the member variable `transl` and use it
                instead. For example, it can used if the translation
                `transl` is predicted from some external model.
                (default=None)
            return_verts: bool, optional
                Return the vertices. (default=True)
            return_full_pose: bool, optional
                Returns the full axis-angle pose vector (default=False)
            Returns
            -------
                output: ModelOutput
                A named tuple of type `ModelOutput`
        '''

        # If no shape and pose parameters are passed along, then use the
        # ones from the module
        global_orient = (global_orient if global_orient is not None else
                         self.global_orient)
        jaw_pose = jaw_pose if jaw_pose is not None else self.jaw_pose
        neck_pose = neck_pose if neck_pose is not None else self.neck_pose

        leye_pose = leye_pose if leye_pose is not None else self.leye_pose
        reye_pose = reye_pose if reye_pose is not None else self.reye_pose

        betas = betas if betas is not None else self.betas
        expression = expression if expression is not None else self.expression

        apply_trans = transl is not None or hasattr(self, 'transl')
        if transl is None:
            if hasattr(self, 'transl'):
                transl = self.transl

        full_pose = torch.cat(
            [global_orient, neck_pose, jaw_pose, leye_pose, reye_pose], dim=1)

        batch_size = max(betas.shape[0], global_orient.shape[0],
                         jaw_pose.shape[0])
        # Concatenate the shape and expression coefficients
        scale = int(batch_size / betas.shape[0])
        if scale > 1:
            betas = betas.expand(scale, -1)
        shape_components = torch.cat([betas, expression], dim=-1)
        shapedirs = torch.cat([self.shapedirs, self.expr_dirs], dim=-1)

        vertices, joints = lbs(shape_components, full_pose, self.v_template,
                               shapedirs, self.posedirs,
                               self.J_regressor, self.parents,
                               self.lbs_weights, pose2rot=pose2rot,
                               )

        lmk_faces_idx = self.lmk_faces_idx.unsqueeze(
            dim=0).expand(batch_size, -1).contiguous()
        lmk_bary_coords = self.lmk_bary_coords.unsqueeze(dim=0).repeat(
            batch_size, 1, 1)
        if self.use_face_contour:
            lmk_idx_and_bcoords = find_dynamic_lmk_idx_and_bcoords(
                vertices, full_pose, self.dynamic_lmk_faces_idx,
                self.dynamic_lmk_bary_coords,
                self.neck_kin_chain,
                pose2rot=True,
            )
            dyn_lmk_faces_idx, dyn_lmk_bary_coords = lmk_idx_and_bcoords
            lmk_faces_idx = torch.cat([lmk_faces_idx,
                                       dyn_lmk_faces_idx], 1)
            lmk_bary_coords = torch.cat(
                [lmk_bary_coords.expand(batch_size, -1, -1),
                 dyn_lmk_bary_coords], 1)

        landmarks = vertices2landmarks(vertices, self.faces_tensor,
                                       lmk_faces_idx,
                                       lmk_bary_coords)

        # Add any extra joints that might be needed
        joints = self.vertex_joint_selector(vertices, joints)
        # Add the landmarks to the joints
        joints = torch.cat([joints, landmarks], dim=1)

        # Map the joints to the current dataset
        if self.joint_mapper is not None:
            joints = self.joint_mapper(joints=joints, vertices=vertices)

        if apply_trans:
            joints += transl.unsqueeze(dim=1)
            vertices += transl.unsqueeze(dim=1)

        output = FLAMEOutput(vertices=vertices if return_verts else None,
                             joints=joints,
                             betas=betas,
                             expression=expression,
                             global_orient=global_orient,
                             neck_pose=neck_pose,
                             jaw_pose=jaw_pose,
                             full_pose=full_pose if return_full_pose else None)
        return output

class FLAMELayer(FLAME):
    def __init__(self, *args, **kwargs) -> None:
        ''' FLAME as a layer model constructor '''
        super(FLAMELayer, self).__init__(
            create_betas=False,
            create_expression=False,
            create_global_orient=False,
            create_neck_pose=False,
            create_jaw_pose=False,
            create_leye_pose=False,
            create_reye_pose=False,
            *args,
            **kwargs)

    def forward(
        self,
        betas: Optional[Tensor] = None,
        global_orient: Optional[Tensor] = None,
        neck_pose: Optional[Tensor] = None,
        transl: Optional[Tensor] = None,
        expression: Optional[Tensor] = None,
        jaw_pose: Optional[Tensor] = None,
        leye_pose: Optional[Tensor] = None,
        reye_pose: Optional[Tensor] = None,
        return_verts: bool = True,
        return_full_pose: bool = False,
        pose2rot: bool = True,
        **kwargs
    ) -> FLAMEOutput:
        '''
        Forward pass for the SMPLX model
            Parameters
            ----------
            global_orient: torch.tensor, optional, shape Bx3x3
                Global rotation of the body. Useful if someone wishes to
                predicts this with an external model. It is expected to be in
                rotation matrix format. (default=None)
            betas: torch.tensor, optional, shape BxN_b
                Shape parameters. For example, it can used if shape parameters
                `betas` are predicted from some external model.
                (default=None)
            expression: torch.tensor, optional, shape BxN_e
                If given, ignore the member variable `expression` and use it
                instead. For example, it can used if expression parameters
                `expression` are predicted from some external model.
            jaw_pose: torch.tensor, optional, shape Bx3x3
                Jaw pose. It should either joint rotations in
                rotation matrix format.
            transl: torch.tensor, optional, shape Bx3
                Translation vector of the body.
                For example, it can used if the translation
                `transl` is predicted from some external model.
                (default=None)
            return_verts: bool, optional
                Return the vertices. (default=True)
            return_full_pose: bool, optional
                Returns the full axis-angle pose vector (default=False)
            Returns
            -------
                output: ModelOutput
                A named tuple of type `ModelOutput`
        '''
        device, dtype = self.shapedirs.device, self.shapedirs.dtype
        if global_orient is None:
            batch_size = 1
            global_orient = torch.eye(3, device=device, dtype=dtype).view(
                1, 1, 3, 3).expand(batch_size, -1, -1, -1).contiguous()
        else:
            batch_size = global_orient.shape[0]
        if neck_pose is None:
            neck_pose = torch.eye(3, device=device, dtype=dtype).view(
                1, 1, 3, 3).expand(batch_size, 1, -1, -1).contiguous()
        if jaw_pose is None:
            jaw_pose = torch.eye(3, device=device, dtype=dtype).view(
                1, 1, 3, 3).expand(batch_size, -1, -1, -1).contiguous()
        if leye_pose is None:
            leye_pose = torch.eye(3, device=device, dtype=dtype).view(
                1, 1, 3, 3).expand(batch_size, -1, -1, -1).contiguous()
        if reye_pose is None:
            reye_pose = torch.eye(3, device=device, dtype=dtype).view(
                1, 1, 3, 3).expand(batch_size, -1, -1, -1).contiguous()
        if betas is None:
            betas = torch.zeros([batch_size, self.num_betas],
                                dtype=dtype, device=device)
        if expression is None:
            expression = torch.zeros([batch_size, self.num_expression_coeffs],
                                     dtype=dtype, device=device)
        if transl is None:
            transl = torch.zeros([batch_size, 3], dtype=dtype, device=device)

        full_pose = torch.cat(
            [global_orient, neck_pose, jaw_pose, leye_pose, reye_pose], dim=1)

        shape_components = torch.cat([betas, expression], dim=-1)
        shapedirs = torch.cat([self.shapedirs, self.expr_dirs], dim=-1)

        vertices, joints = lbs(shape_components, full_pose, self.v_template,
                               shapedirs, self.posedirs,
                               self.J_regressor, self.parents,
                               self.lbs_weights, pose2rot=False,
                               )

        lmk_faces_idx = self.lmk_faces_idx.unsqueeze(
            dim=0).expand(batch_size, -1).contiguous()
        lmk_bary_coords = self.lmk_bary_coords.unsqueeze(dim=0).repeat(
            batch_size, 1, 1)
        if self.use_face_contour:
            lmk_idx_and_bcoords = find_dynamic_lmk_idx_and_bcoords(
                vertices, full_pose, self.dynamic_lmk_faces_idx,
                self.dynamic_lmk_bary_coords,
                self.neck_kin_chain,
                pose2rot=False,
            )
            dyn_lmk_faces_idx, dyn_lmk_bary_coords = lmk_idx_and_bcoords
            lmk_faces_idx = torch.cat([lmk_faces_idx,
                                       dyn_lmk_faces_idx], 1)
            lmk_bary_coords = torch.cat(
                [lmk_bary_coords.expand(batch_size, -1, -1),
                 dyn_lmk_bary_coords], 1)

        landmarks = vertices2landmarks(vertices, self.faces_tensor,
                                       lmk_faces_idx,
                                       lmk_bary_coords)

        # Add any extra joints that might be needed
        joints = self.vertex_joint_selector(vertices, joints)
        # Add the landmarks to the joints
        joints = torch.cat([joints, landmarks], dim=1)

        # Map the joints to the current dataset
        if self.joint_mapper is not None:
            joints = self.joint_mapper(joints=joints, vertices=vertices)

        joints += transl.unsqueeze(dim=1)
        vertices += transl.unsqueeze(dim=1)

        output = FLAMEOutput(vertices=vertices if return_verts else None,
                             joints=joints,
                             betas=betas,
                             expression=expression,
                             global_orient=global_orient,
                             neck_pose=neck_pose,
                             jaw_pose=jaw_pose,
                             full_pose=full_pose if return_full_pose else None)
        return output


# Add key signature list
# Seperate GUIs
#
import sys
import mido
import serial
import asyncio
import concurrent
import os
import numpy as np
from PyQt5.QtWidgets import QMainWindow,QApplication,QPushButton,QWidget,QAction,QTabWidget,QLayout,QVBoxLayout,QHBoxLayout,QLabel,QComboBox,QSlider,QTextEdit,QLineEdit,QGridLayout,QDesktopWidget,QSizePolicy,QGraphicsOpacityEffect
from PyQt5.QtGui import QIcon,QColor,QPalette,QFont,QPixmap,QPainter,QPen,QBrush,QLinearGradient
from PyQt5.QtCore import pyqtSlot,Qt,QSize,QRect,QTimer
from PyQt5.QtTest import QTest

class App(QMainWindow):

    def __init__(self):
        super().__init__()
        self.title = 'Vocal Harmonizer'
        self.setWindowTitle(self.title)
        self.p = self.palette()
        self.setStyleSheet('background-color: rgb(42,41,46); color: rgb(200,209,218)')
        self.central_widget = Central(self)
        self.setCentralWidget(self.central_widget)
        self.resize(1920,400)  # resolution for mac is 1920 x 1200
        self.central_widget.open_midi_port()
        self.show()

class Central(QWidget):

    def __init__(self, parent):
        super().__init__(parent)
        self.layout = QVBoxLayout(self)
        self.window_width = QDesktopWidget().screenGeometry().width()
        self.window_height = QDesktopWidget().screenGeometry().height()
        self.remove_background = 'background-color: rgb(0,0,0,0); border-width: 0px; border-radius: 0px;'
        self.format_button = 'background-color: rgb(247,247,247);'
        self.p = self.palette()
        self.p.setColor(self.backgroundRole(), QColor(127,255,127))
        self.purple = QColor(93,107,68)
        self.blue = QColor(50,209,255)
        self.green = QColor(104,216,196)
        self.orange = QColor(251,165,125)
        self.magenta = QColor(157,39,89)
        self.yellow = QColor(177,176,148)
        self.base_key = 60
        self.starting_key_column = 4
        self.major_pos = [0,3]
        self.minor_pos = [1,3]
        self.chord_pos = [4,0]
        self.prev_midi_status = -1
        self.chord_harmonies = [False,False,False,False,False,False,False,False,False]
        self.ser = serial.Serial()
        self.midi_port_open = 0;
        self.midi_device_buttons = [QPushButton(" "), QPushButton(" "), QPushButton(" "), QPushButton(" ")]
        self.connectBoard()
        self.init_harmonizer()
        self.sendResetToBoard()
        self.layout.setSpacing(self.window_height/20)
        self.layout.addLayout(self.harmonizer_layout)
        self.layout.addStretch()
        self.layout.setContentsMargins(self.window_width/50,self.window_width/50,self.window_height/50,self.window_height/50)
        self.setLayout(self.layout)
        self.opacity = 0.9
        self.timer = QTimer()
        self.timer.setInterval(500)
        self.timer.timeout.connect(self.board_reset_check)
        self.timer.start()

    def board_reset_check(self):
        if self.ser.isOpen():
            try:
                serial_msg = self.ser.read() #.decode('ascii')
                s = list(serial_msg)
                if (len(s) > 0):
                    if (s[0] < 128):
                        status = serial_msg.decode('ascii')
                        if (status == "0"):
                            self.restartGUI()
            except:
                self.ser.close()

    def lambdaChooseKey(self, m):
        return lambda: self.chooseKey(m)

    def lambdaChangeChordHarmonies(self, m):
        return lambda: self.changeChordHarmonies(m)

    def init_harmonizer(self):
        self.gray_text = 'color: rgb(200,209,218)'

        # Create layout to encapsulate harmonizer portion
        self.harmonizer_layout = QGridLayout()

        # Create grid layout for user harmonizer control
        self.harmonizer_layout.setSpacing(10)

        # Look for devices and create label to display current device
        self.device_list = mido.get_input_names()
        self.dev_lbl = QLabel()
        self.dev_lbl.setStyleSheet(self.remove_background + "; color: rgb(59,202,243,244)")
        if self.device_list:
            self.dev_lbl.setText(mido.get_input_names()[0])  # Set label to first device in list
        else:
            self.dev_lbl.setText('No Device')  # Set default label if not device is found
        self.dev_lbl.setFont(QFont('Calibri Light',64))  # Edit label font
        self.dev_lbl.setMaximumSize(self.dev_lbl.minimumSizeHint().width()*1.2,self.dev_lbl.minimumSizeHint().height()*1.3)
        self.harmonizer_layout.addWidget(self.dev_lbl,0,0,1,5,Qt.AlignLeft)

        # Create labels for all keys
        self.key_array = ['C','C#/Db','D','D#/Eb','E','F','F#/Gb','G','G#/Ab','A','A#/Bb','B']
        self.key_buttons = [QPushButton(" ")] * 12
        row = 0
        column = self.starting_key_column
        i = 0
        for key in self.key_array:
            self.key_buttons[i] = QPushButton(key)
            self.key_buttons[i].setStyleSheet(self.remove_background + "; color: rgb(200,209,218,50)")
            self.key_buttons[i].setFont(QFont('Calibri Light',46))
            self.key_buttons[i].setMaximumSize(self.key_buttons[i].minimumSizeHint().width()*1.2,self.key_buttons[i].minimumSizeHint().height())
            self.key_buttons[i].clicked.connect(self.lambdaChooseKey(i))
            self.harmonizer_layout.addWidget(self.key_buttons[i],row,column,Qt.AlignCenter)
            i = i + 1
            if column < self.starting_key_column+3:
                column = column + 1
            else:
                column = self.starting_key_column
                row = row + 1

        # Major key
        self.major_button = QPushButton('Major')
        self.major_button.setStyleSheet(self.remove_background + "; color: rgb(59,202,243,244)")
        self.major_button.setFont(QFont('Calibri Light',46))
        self.major_button.setMaximumSize(self.major_button.minimumSizeHint().width()*1.2,self.major_button.minimumSizeHint().height())
        self.major_button.clicked.connect(lambda: self.chooseKeyMode("Major"))
        self.harmonizer_layout.addWidget(self.major_button,self.major_pos[0],self.major_pos[1],Qt.AlignCenter)

        # Minor Key
        self.minor_button = QPushButton('Minor')
        self.minor_button.setStyleSheet(self.remove_background + "; color: rgb(200,209,218,50)")
        self.minor_button.setFont(QFont('Calibri Light',46))
        self.minor_button.setMaximumSize(self.minor_button.minimumSizeHint().width()*1.2,self.minor_button.minimumSizeHint().height())
        self.minor_button.clicked.connect(lambda: self.chooseKeyMode("Minor"))
        self.harmonizer_layout.addWidget(self.minor_button,self.minor_pos[0],self.minor_pos[1],Qt.AlignCenter)

         # Show default key signature
        self.current_key_button = self.key_buttons[4]
        self.setOriginalKey()

        button = QLabel('Chord Harmonies')
        button.setStyleSheet(self.remove_background + "; color: rgb(59,202,243,244)")
        button.setFont(QFont('Calibri Light',46))
        button.setMaximumSize(button.minimumSizeHint().width(),button.minimumSizeHint().height())
        self.harmonizer_layout.addWidget(button,self.chord_pos[0]-1,0,1,2,Qt.AlignLeft | Qt.AlignTop)

        button = QLabel(' ')
        button.setStyleSheet(self.remove_background + "; color: rgb(200,209,218,244)")
        button.setFont(QFont('Calibri Light',46))
        button.setMaximumSize(button.minimumSizeHint().width()*1.2,button.minimumSizeHint().height())
        self.harmonizer_layout.addWidget(button,self.chord_pos[0]-1,0,Qt.AlignCenter)

        #Show 9 harmony indicators
        self.harmony_array = ['Octave Below ', '3rd Below ', '5th Below ', '6th Below ', '3rd Above ', '5th Above ', '6th Above ', 'Octave Above ', 'Autotune']
        self.harmony_buttons = [QPushButton(" ")] * 9
        row = self.chord_pos[0]
        column = self.chord_pos[1]
        i = 0
        for harmony in self.harmony_array:
            self.harmony_buttons[i] = QPushButton(harmony)
            self.harmony_buttons[i].setStyleSheet(self.remove_background + "; color: rgb(200,209,218,50)")
            self.harmony_buttons[i].setFont(QFont('Calibri Light',36))
            self.harmony_buttons[i].setMaximumSize(self.harmony_buttons[i].minimumSizeHint().width(),self.harmony_buttons[i].minimumSizeHint().height())
            self.harmony_buttons[i].clicked.connect(self.lambdaChangeChordHarmonies(i))
            self.harmonizer_layout.addWidget(self.harmony_buttons[i],row,column,Qt.AlignCenter)
            column = column + 1
            i = i + 1

        # Create search button
        self.search_button = QPushButton('Search')
        self.search_button.setFont(QFont('Calibri',14))
        self.search_button.setStyleSheet('background-color: rgb(127,127,127,127); border-width: 2px; border-style: outset; border-radius: 5px; padding: 2px; border-color: transparent')
        self.search_button.setMinimumSize(self.search_button.minimumSizeHint().width()*2,self.search_button.minimumSizeHint().height())
        self.search_button.pressed.connect(lambda: self.search_button.setStyleSheet('background-color: rgb(192,192,192,127); border-width: 2px; border-style: outset; border-radius: 5px; padding: 2px; border-color: transparent'))
        self.search_button.released.connect(lambda: self.search_button.setStyleSheet('background-color: rgb(127,127,127,127); border-width: 2px; border-style: outset; border-radius: 5px; padding: 2px; border-color: transparent'))
        self.search_button.clicked.connect(self.searchMidi)
        self.harmonizer_layout.addWidget(self.search_button,1,0,Qt.AlignLeft)

        # Create restart button
        self.restart_button = QPushButton('Restart')
        self.restart_button.setFont(QFont('Calibri',14))
        self.restart_button.setStyleSheet('background-color: rgb(127,127,127,127); border-width: 2px; border-style: outset; border-radius: 5px; padding: 2px; border-color: transparent')
        self.restart_button.setMinimumSize(self.restart_button.minimumSizeHint().width()*2,self.restart_button.minimumSizeHint().height())
        self.restart_button.pressed.connect(lambda: self.restart_button.setStyleSheet('background-color: rgb(192,192,192,127); border-width: 2px; border-style: outset; border-radius: 5px; padding: 2px; border-color: transparent'))
        self.restart_button.released.connect(lambda: self.restart_button.setStyleSheet('background-color: rgb(127,127,127,127); border-width: 2px; border-style: outset; border-radius: 5px; padding: 2px; border-color: transparent'))
        self.restart_button.clicked.connect(self.restartGUI)
        self.harmonizer_layout.addWidget(self.restart_button,1,1,Qt.AlignLeft)

    def searchMidi(self):
        if self.midi_port_open:
            self.midi_port.close()
            self.midi_port_open = False
        midi_device = mido.get_input_names()
        for i in range(len(midi_device)):
            self.midi_device_buttons[i] = QPushButton(midi_device[i])
            self.midi_device_buttons[i].setFont(QFont('Calibri',14))
            self.midi_device_buttons[i].setStyleSheet('background-color: rgb(127,127,127,127); border-width: 2px; border-style: outset; border-radius: 5px; padding: 2px; border-color: transparent')
            self.midi_device_buttons[i].setMinimumSize(self.midi_device_buttons[i].minimumSizeHint().width()*2,self.midi_device_buttons[i].minimumSizeHint().height())
            self.midi_device_buttons[i].pressed.connect(lambda: self.midi_device_buttons[i].setStyleSheet('background-color: rgb(192,192,192,127); border-width: 2px; border-style: outset; border-radius: 5px; padding: 2px; border-color: transparent'))
            self.midi_device_buttons[i].released.connect(lambda: self.midi_device_buttons[i].setStyleSheet('background-color: rgb(127,127,127,127); border-width: 2px; border-style: outset; border-radius: 5px; padding: 2px; border-color: transparent'))
            self.midi_device_buttons[i].clicked.connect(lambda: self.midi_device_clicked(self.midi_device_buttons[i])) #self.open_midi_port(midi_device[i]); search_button.deleteLater)
            self.harmonizer_layout.addWidget(self.midi_device_buttons[i],i,2,Qt.AlignLeft)

    def midi_device_clicked(self, midi_button):
        self.open_midi_port(midi_button.text())
        for i in range(len(self.midi_device_buttons)):
            self.midi_device_buttons[0].deleteLater()

    # callback for restart button
    def restartGUI(self):
        self.connectBoard()

        self.resetKeyAndHarmonies()

        if self.midi_port_open:
            self.midi_port.close()
            self.midi_port_open = False
        self.open_midi_port()

    def sendResetToBoard(self):
        if self.ser.isOpen():
            self.ser.write([255, 255, 255])

    def setOriginalKey(self):
        old_key_button = self.current_key_button
        old_key_button.setStyleSheet(self.remove_background + "; color: rgb(200,209,218,50)")
        self.current_key_midi_num = 64 # E
        self.current_key_mode = "Major"
        self.current_key_button = self.key_buttons[4]
        self.current_key_button.setStyleSheet(self.remove_background + "; color: rgb(59,202,243,244)")

        # Dim minor
        self.minor_button.setStyleSheet(self.remove_background + "; color: rgb(200,209,218,50)")

        # Highlight major
        self.major_button.setStyleSheet(self.remove_background + "; color: rgb(59,202,243,244)")

    def resetKeyAndHarmonies(self):
        self.setOriginalKey()

        for i in range(0,9):
            self.chord_harmonies[i] = False

        row = self.chord_pos[0]
        column = self.chord_pos[1]
        for i in range(len(self.harmony_buttons)): #.harmony_array:
            harmony_button = self.harmony_buttons[i] #self.harmonizer_layout.itemAtPosition(row,column).widget()
            harmony_button.setStyleSheet(self.remove_background + "; color: rgb(200,209,218,50)")
            column = column + 1

        self.sendResetToBoard()

    def chooseKeyMode(self, mode):
        if self.ser.isOpen():
            button = self.sender()
            button.setStyleSheet(self.remove_background + "; color: rgb(59,202,243,244)")
            if(mode == "Major"):
                if(self.current_key_mode != "Major"):
                    # dim minor button
                    self.minor_button.setStyleSheet(self.remove_background + "; color: rgb(200,209,218,50)")
                    self.current_key_mode = "Major"
                    self.current_key_midi_num -= 3
                    self.sendKeyChange(self.current_key_midi_num)
            elif(mode == "Minor"):
                if (self.current_key_mode != "Minor"):
                    # dim major button
                    self.major_button.setStyleSheet(self.remove_background + "; color: rgb(200,209,218,50)")
                    self.current_key_mode = "Minor"
                    self.current_key_midi_num += 3
                    self.sendKeyChange(self.current_key_midi_num)

    def chooseKey(self, key_offset):
        if self.ser.isOpen():
            old_key_button = self.current_key_button
            old_key_button.setStyleSheet(self.remove_background + "; color: rgb(200,209,218,50)")
            self.current_key_button = self.sender()
            self.current_key_button.setStyleSheet(self.remove_background + "; color: rgb(59,202,243,244)")
            self.current_key_midi_num = self.base_key + key_offset
            if(self.current_key_mode == "Minor"):
                self.current_key_midi_num +=3
            self.sendKeyChange(self.current_key_midi_num)

    def sendKeyChange(self, midi_key_num):
        if self.ser.isOpen():
            self.ser.write([255, 255, midi_key_num])
            print(midi_key_num)
        else:
            print('Cannot send key')

    def changeChordHarmonies(self, chord_idx):
        if self.ser.isOpen():
            self.chord_harmonies[chord_idx] ^= True
            if (self.chord_harmonies[chord_idx]):
                self.harmony_buttons[chord_idx].setStyleSheet(self.remove_background + "; color: rgb(59,202,243,244)")
            else:
                self.harmony_buttons[chord_idx].setStyleSheet(self.remove_background + "; color: rgb(200,209,218,50)")
            self.ser.write([176, 32, 0]) # tells board that it's a button from one of the channels (MIDI keyboard uses + and - buttons sometimes and sends same message)
            self.ser.write([192, chord_idx])

    def connectBoard(self):
        if self.ser.isOpen():
            self.ser.close()
        if (os.path.exists("/dev/tty.usbmodem1462")):
            self.ser = serial.Serial(port='/dev/tty.usbmodem1462', baudrate=115200, bytesize=serial.EIGHTBITS, parity=serial.PARITY_NONE, timeout=0.001)
        elif (os.path.exists("/dev/tty.usbmodem1442")):
            self.ser = serial.Serial(port='/dev/tty.usbmodem1442', baudrate=115200, bytesize=serial.EIGHTBITS, parity=serial.PARITY_NONE, timeout=0.001)
        elif (os.path.exists("/dev/tty.usbserial-A904RDA2")):
            self.ser = serial.Serial(port='/dev/tty.usbserial-A904RDA2', baudrate=115200, bytesize=serial.EIGHTBITS, parity=serial.PARITY_NONE, timeout=0.001)
        elif (os.path.exists("/dev/tty.usbserial-A904RDA3")):
            self.ser = serial.Serial(port='/dev/tty.usbserial-A904RDA3', baudrate=115200, bytesize=serial.EIGHTBITS, parity=serial.PARITY_NONE, timeout=0.001)
        else:
            self.ser = serial.Serial(baudrate=115200, bytesize=serial.EIGHTBITS, parity=serial.PARITY_NONE, timeout=0.001)
            print("Error, no board connected")
            return
        try:
            if(self.ser.isOpen()):
                print("Serial port is open")
            else:
                print("Error, no board connected")
        except:
            print("Error, no board connected")
            return

    def message_callback(self, message):
        if (message.bytes()[0] == 192 and self.prev_midi_status == 176):
            chord_idx = message.bytes()[1]
            if (chord_idx < 9):
                self.chord_harmonies[chord_idx] ^= True
                harmony = self.harmonizer_layout.itemAtPosition(self.chord_pos[0],chord_idx).widget()
                if (self.chord_harmonies[chord_idx]):
                    harmony.setStyleSheet(self.remove_background + "; color: rgb(59,202,243,244)")
                else:
                    harmony.setStyleSheet(self.remove_background + "; color: rgb(200,209,218,50)")

        self.prev_midi_status = message.bytes()[0]
        if self.ser.isOpen():
            self.ser.write(message.bytes())
        print(message.bytes())

    def open_midi_port(self, device=None):
        if(device == None):
            midi_device = mido.get_input_names()
            if (midi_device):
                device = midi_device[0]

        if (device):
            self.midi_port = mido.open_input(device)
            self.midi_port.callback = self.message_callback
            self.midi_port_open = True
            self.dev_lbl.setText(mido.get_input_names()[0])
            print("Midi device connected")
        else:
            self.dev_lbl.setText('No Device')
            print("Error, no midi device connected")

app = QApplication(sys.argv)
ex = App()
ex.setWindowOpacity(0.99)
sys.exit(app.exec_())


import pandas as pd
from numpy import random
import matplotlib.pyplot as plt
random.seed(500)
names = ['Bob','Jessica','Mary','John','Mel']

random_names = [names[random.randint(low=0,high=len(names))] for i in range(1000)]
births = [random.randint(low=0,high=1000) for i in range(1000)]
BabyDataSet = zip(random_names,births)
df = pd.DataFrame(data = BabyDataSet, columns=['Names', 'Births']).groupby('Names').sum()
df.sort(['Births'], ascending=False,inplace=True)
df.plot(kind='bar',legend=False)
plt.show()


import commonmark,re,argparse

import sys
import io
import os
import shutil
import yaml
import time
import datetime

from html import escape

TEXT_MARKDOWN = 'text/markdown'
TEXT_HTML = 'text/html'
TEXT_XML = 'text/xml'
def normalize(s):
   return s.replace('\n',' ')

def literal(s):
   return s.replace('"','\\"')

def isoformat(s):
   return s if type(s)==str else s.isoformat()

def now():
   return datetime.datetime.now().astimezone()

class ArticleConverter:
   def __init__(self,weburi,entryuri):
      self.weburi = weburi
      self.entryuri = [ entryuri ]

   def enter(self,suffix):
      self.entryuri.append(self.entryuri[-1] + suffix)

   def leave(self):
      return self.entryuri.pop()

   def toArticle(self,base,md,html,triples):
      metadata = yaml.load(md,Loader=yaml.Loader)

      summary = ''
      hasTitle = False
      for part in metadata.get('content','').split('\n'):
         if part[0:2]=='# ':
            hasTitle = True
            continue
         if len(part)>0:
            summary = part
            break
      if 'description' not in metadata:
         metadata['description'] = normalize(summary)

      if 'updated' not in metadata:
         metadata['updated'] = now()

      if 'published' not in metadata:
         metadata['published'] = metadata['updated']

      self.toHTML(base,metadata.get('content',''),metadata,html,generateTitle=not hasTitle)
      self.toTurtle(base,metadata,triples)

   def toHTML(self,base,content,metadata,html,generateTitle=False):

      uri = self.weburi + isoformat(metadata.get('published',''))

      print('
'.format(uri),file=html)

      print('',file=html)

      if generateTitle:
         print("
{}
".format(escape(metadata['title'],quote=False)),file=html)

      format = metadata.get('format',TEXT_MARKDOWN)
      if format==TEXT_MARKDOWN:
         print(commonmark.commonmark(content),file=html)
      elif format==TEXT_HTML:
         html.write(content)
      elif format==TEXT_XML:
         self.transformXML(content,html)
      else:
         raise ValueError('Unknown format: {}'.format(format))

      print('
',file=html)

   def transformXML(self,content,output):
      pass

   def toTurtle(self,base,metadata,triples):

      uri = self.weburi + isoformat(metadata['published'])
      basedOn = self.entryuri[-1] + base + '.html'
      #name = base + '.html'

      print('@base <{}> .'.format(uri),file=triples)
      print('@prefix schema: <http://schema.org/> .',file=triples)
      print('<>',file=triples)
      print('   a schema:BlogPosting ;',file=triples)
      print('   schema:genre "blog";',file=triples)
      print('   schema:name "{}";'.format(base),file=triples)
      print('   schema:headline "{}" ;'.format(metadata['title']),file=triples)
      print('   schema:description "{}" ;'.format(literal(metadata['description'])),file=triples)
      print('   schema:datePublished "{}" ;'.format(isoformat(metadata['published'])),file=triples)
      print('   schema:dateModified "{}" ;'.format(isoformat(metadata['updated'])),file=triples)
      print('   schema:isBasedOnUrl "{}";'.format(basedOn),file=triples)
      #print('   schema:hasPart [ a schema:MediaObject; schema:contentUrl "{0}"; schema:fileFormat "text/html"; schema:name "{1}" ] ;'.format(basedOn,name),file=triples)
      if ("keywords" in metadata):
         print('   schema:keywords {} ;'.format(','.join(['"' + m + '"' for m in metadata['keywords']])),file=triples)
      if 'author' in metadata:
         authors = metadata.get('author',[])
         triples.write('   schema:author ')
         for index,author in enumerate(authors if type(authors)==list else [authors]):
            if (index>0):
               triples.write(', ')
            triples.write('[ a schema:Person; schema:name "{}" ]'.format(author))
      triples.write(' .\n')

argparser = argparse.ArgumentParser(description='Article HTML and Turtle Generator')
argparser.add_argument('-f',action='store_true',help='Forces all the files to be regenerated.',dest='force')
argparser.add_argument('--extension',nargs='?',help='The source file extension',default='md')
argparser.add_argument('-o',nargs='?',help='The output directory',dest='outdir')
argparser.add_argument('-w',nargs='?',help='The web uri',dest='weburi',default='http://www.milowski.com/journal/entry/')
argparser.add_argument('-e',nargs='?',help='The entry uri directory',dest='entryuri',default='http://alexmilowski.github.io/milowski-journal/')
argparser.add_argument('dir',nargs=1,help='The directory to process.')
args = argparser.parse_args()
inDir = args.dir[0]
outDir = args.outdir if (args.outdir!=None) else inDir
dirs = [d for d in os.listdir(inDir) if not(d[0]=='.') and os.path.isdir(inDir + '/' + d)]

converter = ArticleConverter(args.weburi,args.entryuri)

extension = '.' + args.extension if args.extension[0]!='.' else args.extension
extension_count = extension.count('.')

for dir in dirs:

   sourceDir = inDir + '/' + dir
   targetDir = outDir + '/' + dir

   if (not(os.path.exists(targetDir))):
      os.makedirs(targetDir)

   converter.enter(dir+'/')

   files = [f for f in os.listdir(sourceDir) if f.endswith(extension) and os.path.isfile(sourceDir + '/' + f)]

   for file in files:

      targetFile = sourceDir + '/' + file
      base = file.rsplit('.',extension_count)[0]
      htmlFile = targetDir + '/' + base + ".html"
      turtleFile = targetDir + '/' + base + ".ttl"

      updatedNeeded = args.force or not(os.path.exists(htmlFile)) or not(os.path.exists(turtleFile))
      if (not(updatedNeeded)):
         btime = os.path.getmtime(targetFile)
         updatedNeeded = btime > os.path.getmtime(htmlFile) or btime > os.path.getmtime(turtleFile)

      if (not(updatedNeeded)):
         continue

      print(file + " → " + htmlFile + ", " + turtleFile)

      md = open(targetFile,"r")
      html = open(htmlFile,"w")
      turtle = open(turtleFile,"w")

      converter.toArticle(base,md,html,turtle)

      md.close()
      html.close()
      turtle.close()

   work = [f for f in os.listdir(sourceDir) if (not f.endswith('.md'))]
   for file in work:
      sourceFile = sourceDir + '/' + file
      targetFile = targetDir + '/' + file
      if os.path.isfile(sourceFile):
         copyNeeded = args.force or not(os.path.exists(targetFile)) or os.path.getmtime(sourceFile) > os.path.getmtime(targetFile)

         if copyNeeded:
            print(file + " → " + targetFile)
            shutil.copyfile(sourceFile,targetFile)
      elif os.path.isdir(sourceFile):
         if os.path.exists(targetFile):
            print('sync tree ' + sourceFile)
            work += [sourceFile + '/' + f for f in os.listdir(sourceFile)]
         else:
            print("copy tree " + sourceFile + " → " + targetFile)
            shutil.copytree(sourceFile,targetFile)

   converter.leave()


# 3.20:  Access tag by parameter using a dict.  Use a dict
# with .find() to specify a 'class' parameter value (e.g.
# {'class': 'story_title'})

from bs4 import BeautifulSoup

scrapee = '../dormouse.html'

text = open(scrapee).read()

soup = BeautifulSoup(text, 'html.parser')


import math
from datetime import date
import pandas_datareader as web
import numpy as np
import pandas as pd
from sklearn.preprocessing import MinMaxScaler
import tensorflow as tf
from tensorflow import keras
from keras.models import Sequential
from keras.layers import Dense, LSTM
import matplotlib.pyplot as plot
plot.style.use('ggplot')

def closing_prices(company):
    print("The program will use 80% of the data to train and the rest will be used for testing.")
    print("Please enter a starting date (ex. 2012-01-01 [yyyy-MM-DD]): ")
    start = input()
    print("Please enter a closing date (blank for today): ")
    end = input()
    if(len(end) == 0):
      end = date.today()
    df = web.DataReader(company, data_source='yahoo', start=start, end=end)
    #Visualize the closing price history
    plot.figure(figsize=(16,8))
    plot.title('Close Price History - {}'.format(company))
    plot.plot(df['Close'])
    plot.xlabel('Date',fontsize=18)
    plot.ylabel('Close Price USD ($)',fontsize=18)
    plot.show()

def test_ml(company):
    print("The program will use 80% of the data to train and the rest will be used for testing.")
    print("Please enter a starting date (ex. 2012-01-01 [yyyy-MM-DD]): ")
    start = input()
    print("Please enter a closing date (blank for today): ")
    end = input()
    if(len(end) == 0):
      end = date.today()
    df = web.DataReader(company, data_source='yahoo', start=start, end=end)
    data = df.filter(['Close'])
    dataset = data.values
    #Compute the number of rows to train the model on
    training_data_len = math.ceil( len(dataset) *.8)
    #Preprocessing the data, scale to 0 thru 1
    scaler = MinMaxScaler(feature_range=(0, 1))
    scaled_data = scaler.fit_transform(dataset)

    #Create the scaled training data set
    train_data = scaled_data[0:training_data_len  , : ]
    x_train=[]
    y_train = []
    for i in range(60,len(train_data)):
        x_train.append(train_data[i-60:i,0]) # Store adjusted stock prices for the last 60 days (not including curr day)
        y_train.append(train_data[i,0])     # Store adjusted stock price for current day

    print("Getting training data...")
    #Convert x_train and y_train to numpy arrays
    x_train, y_train = np.array(x_train), np.array(y_train)
    #Reshape the data into the shape accepted by the LSTM (3-dimensional)
    x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], 1))

    print("Creating model...")
    #Build the LSTM network model with two 50 neurons layers and two dense layers, one with 25 and the other with 1 neuron.
    model = Sequential()
    model.add(LSTM(units=50, return_sequences=True, input_shape=(x_train.shape[1],1)))
    model.add(LSTM(units=50, return_sequences=False))
    model.add(Dense(units=25))
    model.add(Dense(units=1))

    print("Compiling model...")
    model.compile(optimizer='adam', loss='mean_squared_error')

    print("Training model... (This part may take a while)")
    model.fit(x_train, y_train, batch_size=1, epochs=1)

    print("Creating test sets...")
    test_data = scaled_data[training_data_len - 60: , : ]
    #Create the x_test and y_test data sets
    x_test = []
    y_test =  dataset[training_data_len : , : ] #Get rest of data
    for i in range(60,len(test_data)):
        x_test.append(test_data[i-60:i,0])

    #Convert x_test to a numpy array
    x_test = np.array(x_test)

    #Reshape the data into the shape accepted by the LSTM
    x_test = np.reshape(x_test, (x_test.shape[0],x_test.shape[1],1))

    print("Getting predicted values...")
    #Getting the models predicted price values
    predictions = model.predict(x_test)
    predictions = scaler.inverse_transform(predictions)#Undo scaling

    print("Done!")

    #Calculate/Get the value of RMSE
    rmse=np.sqrt(np.mean(((predictions- y_test)**2)))
    print("RSME = {}".format(rmse))

    #Plot/Create the data for the graph
    train = data[:training_data_len]
    valid = data[training_data_len:]
    valid['Predictions'] = predictions
    #Visualize the data
    plot.figure(figsize=(16,8))
    plot.title('Close Price Model - {}'.format(company))
    plot.xlabel('Date', fontsize=18)
    plot.ylabel('Close Price USD ($)', fontsize=18)
    plot.plot(train['Close'])
    plot.plot(valid[['Close', 'Predictions']])
    plot.legend(['Training Data', 'Real Data', 'Predictions'], loc='lower right')
    plot.show()

def predict(company):
    print("The program will attempt to predict tomorrow's closing price.")
    print("Please enter a starting date for training (ex. 2012-01-01 [yyyy-MM-DD]): ")
    start = input()
    end = date.today()
    df = web.DataReader(company, data_source='yahoo', start=start, end=end)
    data = df.filter(['Close'])
    dataset = data.values
    #Model will train on entire dataset
    training_data_len = len(dataset)
    #Preprocessing the data, scale to 0 thru 1
    scaler = MinMaxScaler(feature_range=(0, 1))
    scaled_data = scaler.fit_transform(dataset)

    #Create the scaled training data set
    train_data = scaled_data[0:training_data_len  , : ]
    x_train=[]
    y_train = []
    for i in range(60,len(train_data)):
        x_train.append(train_data[i-60:i,0]) # Store adjusted stock prices for the last 60 days (not including curr day)
        y_train.append(train_data[i,0])     # Store adjusted stock price for current day

    print("Getting training data...")
    #Convert x_train and y_train to numpy arrays
    x_train, y_train = np.array(x_train), np.array(y_train)
    #Reshape the data into the shape accepted by the LSTM (3-dimensional)
    x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], 1))

    print("Creating model...")
    #Build the LSTM network model with two 50 neurons layers and two dense layers, one with 25 and the other with 1 neuron.
    model = Sequential()
    model.add(LSTM(units=50, return_sequences=True, input_shape=(x_train.shape[1],1)))
    model.add(LSTM(units=50, return_sequences=False))
    model.add(Dense(units=25))
    model.add(Dense(units=1))

    print("Compiling model...")
    model.compile(optimizer='adam', loss='mean_squared_error')

    print("Training model... (This part may take a while)")
    model.fit(x_train, y_train, batch_size=1, epochs=1)

    print("Creating test set...")
    last_60_days = data[-60:].values
    last_60_days_scaled = scaler.transform(last_60_days)
    X_test = []
    X_test.append(last_60_days_scaled)
    #Convert the X_test data set to a numpy array
    X_test = np.array(X_test)
    #Reshape the data
    X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))

    print("Getting predicted values...")
    pred_price = model.predict(X_test)

    print("Done!")
    pred_price = scaler.inverse_transform(pred_price)
    print("Tomorrow's predicted close price for {comp} is {price}".format(comp=company, price=pred_price[0][0]))

def main():
    print("Welcome to Stock Price Predictor! This program will use ML to predict stock prices")
    print("Please enter the trading symbol you would like to examine (ex TSLA or ^DJI): ")
    company = input()
    try:
        df = web.DataReader(company, data_source='yahoo', start="2012-01-01", end="2012-01-05")
    except:
        print("Error trading symbol not valid")
        sys.exit()
    print("What would you like to do? *Note: all starting days must be at least 60 days before the end date")
    print("  1. View Closing Prices (History)")
    print("  2. Test the ML Model")
    print("  3. Predict Stock Price")
    choice = input("Option: ")
    if(choice == "1"):
        closing_prices(company)
    elif(choice == "2"):
        test_ml(company)
    elif(choice == "3"):
        predict(company)

    else:
        print("Guess you haven't made up your mind. Please try again later.")

if __name__ == "__main__":
    main()


"""A collection of features that characterize a PSD."""

import pickle
import numpy as np
from collections import defaultdict

from scipy.signal import find_peaks
from scipy.integrate import simpson
from openseize.core.arraytools import slice_along_axis
from openseize.spectra.metrics import power_norm

def fetch_psds(data, geno, condition, genos=['ube3a', 'stxbp1']):
    """Returns the names, counts, frequencies & PSDs of animals with matching
    genotype & condition from a pickled list of dicts stored at path.

    Args:
        data:
            A list of dicts one per geno in genos where each dict has structure:
            {animal_i: {(condition_tuple)_j: (psd_tuple)_ij}
            where i runs over animals in this geno and condition runs over possible
            detection condition tuples (eg. [(awake,), (awake, annotated), ...]
        geno:
            A string name of a genotype to return psd results from.
        condition:
            A 2-tuple condition key to return cnts, freqs and psds for.
        genos:
            The order of the genotypes of the pickled list stored at path.

    Returns:
        names:
            A list of animal names in the same order as cnts, freqs & psds
        cnts:
            A 1-D array of counts, one per animal of that matches genotype & state
        freqs:
            A 1-D array of psd frequencies. Note assumption is that all psds
            were computed at the SAME frequencies.
        psds:
            A 3-D array of shape animals x channels x frequencies.
    """

    names, cnts, freqs, psds = [], [], [], []
    geno_idx = genos.index(geno)
    for animal, state_dict in data[geno_idx].items():
        for state_tuple, (cnt, freqs, psd) in state_dict.items():
            if condition == state_tuple:
                names.append(animal)
                cnts.append(cnt)
                psds.append(psd)

    cnts = np.array(cnts)
    psds = np.stack(psds)
    return names, cnts, freqs, psds

def filter_conditions(data, filterby='awake'):
    """Returns a list of conditions that contain the filterby state string.

    Args:
        data:
            A list of dicts one per geno in genos where each dict has structure:
            {animal_i: {(state_tuple)_j: (psd_tuple)_ij}
            where i runs over animals in this geno and condition runs over possible
            detection condition tuples (eg. [(awake,), (awake, annotated), ...]
        filterby:
            A string that must be a part of data's condition tuple keys in order
            to be returned.

    Returns:
        A list of condition tuples.
    """

    state_dict = list(data[0].values())[0]
    return [tup for tup in state_dict if filterby in tup]

def peak_counts(psds, **kwargs):
    """Returns the number of peaks feature from 3-D PSD data.

    Args:
        psds:
            An animals x channels x frequencies array of PSD values.

    Returns:
        A 2-D array of shape chs x animals containing peak counts.
    """

    results = np.zeros((psds.shape[1], psds.shape[0]))
    for mouse_idx, arr in enumerate(psds):
        for ch_idx, signal in enumerate(arr):

            peaks, _ = find_peaks(signal, **kwargs)
            results[ch_idx, mouse_idx] = len(peaks)

    return results

def max_height(psds, **kwargs):
    """Returns the height of the largest prominence peak feature in psds for
    each animal and channel.

    Args:
        psds:
            An animals x channels x frequencies array of PSD values.

    Returns:
        A 2-D array of shape chs x animals containing heights.
    """

    results = np.zeros((psds.shape[1], psds.shape[0]))
    for mouse_idx, arr in enumerate(psds):
        for ch_idx, signal in enumerate(arr):

            locs, info = find_peaks(signal, **kwargs)
            idx = np.argmax(info['prominences'])
            freq_idx = locs[idx]
            results[ch_idx, mouse_idx] = psds[mouse_idx, ch_idx, freq_idx]

    return results

def max_prominence(psds, **kwargs):
    """Returns the prominence of the largest prominence peak feature in psds
    for each animal and channel.

    Args:
        psds:
            An animals x channels x frequencies array of PSD values.

    Returns:
        A 2-D array of shape chs x animals containing max peak prominences.
    """

    results = np.zeros((psds.shape[1], psds.shape[0]))
    for mouse_idx, arr in enumerate(psds):
        for ch_idx, signal in enumerate(arr):

            _, info = find_peaks(signal, **kwargs)
            results[ch_idx, mouse_idx] = np.max(info['prominences'])

    return results

def max_width(psds, **kwargs):
    """Returns the width of the largest prominence peak feature in psds for each
    animal and channel.

    Args:
        psds:
            An animals x channels x frequencies array of PSD values.

    Returns:
        A 2-D array of shape chs x animals containing max peak widths.
    """

    results = np.zeros((psds.shape[1], psds.shape[0]))
    for mouse_idx, arr in enumerate(psds):
        for ch_idx, signal in enumerate(arr):

            _, info = find_peaks(signal, **kwargs)
            idx = np.argmax(info['prominences'])
            results[ch_idx, mouse_idx] = info['widths'][idx]

    return results

def max_location(psds, **kwargs):
    """Returns the index of the max prominence peak feature in psds for each
    animal and channel.

    Args:
        psds:
            An animals x channels x frequencies array of PSD values.

    Returns:
        A 2-D array of shape chs x animals containing frequencies of max
        prominent peaks.
    """

    results = np.zeros((psds.shape[1], psds.shape[0]))
    for mouse_idx, arr in enumerate(psds):
        for ch_idx, signal in enumerate(arr):

            locs, info = find_peaks(signal, **kwargs)
            idx = np.argmax(info['prominences'])
            results[ch_idx, mouse_idx] = locs[idx]

    return results

def area(psds, **kwargs):
    """Returns the area below each PSD in psds.

    Args:
        psds:
            An animals x channels x frequencies array of PSD values.

    Returns:
        A 2-D array of shape chs x animals containing areas.

    Note:
        The spacing of integration points is 1 meaning the frequency resolution
        is ignored. This is fine since similarity is a comparison and the actual
        area units are not used.
    """

    x = simpson(psds, axis=-1)
    # all features return chs x animals
    return x.T

def featurize_psds(psds, freqs, features, start, stop, **kwargs):
    """Transforms a 3-D PSD array into a 3-D feature array for frequencies
    between start and stop.

    Args:
        psds:
            An animals x channels x frequencies array of PSD values.
        freqs:
            A 1-D array of frequencies for which PSDs were measured.
        *features:
            A list of frozen callables whose only free variable is psds.
        start:
            The start frequency over which to measure features. Must be an
            element of freqs.
        stop:
            The stop frequency over which to measure features. Must be an
            element of freqs.

    Returns:
        A len(features) x len(channels) x len(animals) feature array.
    """

    frequencies = list(freqs)
    a, b = [frequencies.index(f) for f in (start, stop)]
    x = slice_along_axis(psds, a, b, axis=-1)

    result = []
    for feature in features:
        result.append(feature(x))

    return np.stack(result)

def featurize(path, geno, state, features, genos=['ube3a', 'stxbp1'], start=0,
              stop=40):
    """Featurizes all the PSDs pickled at path for a given genotype and
    containing state in the condition.

    Args:
        path:
            String or Path instance to a pickle containing a list of PSD dicts
            with the following structures:
            {animal_i: {(state_tuple)_j: (psd_tuple)_ij}
            where i runs over animals in this geno and condition runs over possible
            detection condition tuples (eg. [(awake,), (awake, annotated), ...]
        geno:
            The genotype for which similarities will be determined, Must be one
            of genos.
        features:
            A list of callables that accept only a 3-D PSD ndarray with shape
            animals x channels x frequencies.
        genos:
            The order of the genotypes in the pickled list stored at path.
        start & stop:
            The start to stop frequencies over which features will be measured.
            Must be an actual frequency in the pickled PSD data.

    Returns:
        A 2-el tuple with the first element being a dict keyed on conditions
        with an ndarray of features for values. The shape of this array will be
        len(features) x len(channels) x len(animals). The tuples second element
        is a list of the string names of the animals in the same order as the
        last axis of the PSD ndarrays.
    """

    with open(path, 'rb') as infile:
        data = pickle.load(infile)

    result = {}
    for condition in filter_conditions(data, filterby=state):
        # fetch the PSDs for each condition in filtered conditions
        # PSDs.shape = (animals, channels, frequencies)
        names, cnts, freqs, psds = fetch_psds(data, geno, condition, genos=genos)

        # featurize PSDs and store
        # featurized.shape = (features, channels, animals)
        result[condition] = featurize_psds(psds, freqs, features, start=start,
                                          stop=stop)

    return result, names

def similarity(featurized, relative_to=('awake', 'annote')):
    """Returns a dict of cosine similarities one per condition in featurized
    relative to supplied condition.

    Args:
        featurized:
            A dict containing condition keys and correpsonding 3-D feature
            arrays of shape (features, channels, animals) (see featurize)
        relative_to:
            A tuple indicating which vectors in featurized each featurized
            vector should be compared against for cosine similarity.

    Returns:
        A dict with the same condition keys as featurized and correponding 1-D
        array of cosine similarities, the average of the cosine similarities for
        each channel in featurized.
    """

    comparator = featurized[relative_to]

    # use matrix multiplication to compute all dot products at once:
    # diag(A x B) / sqrt(sum(A**2, feature_axis) * sum(B**2, feature_axis))
    # Jasmine seek to understand how numpy does high-d matrix multiplication

    # move comparators animal axis to last -> (animals, features, chs)
    y = np.moveaxis(comparator, -1, 0)
    #print(f'y shape is: {y.shape}')

    result = {}
    for condition, arr in featurized.items():

        # swap the featurized fea
        x = np.swapaxes(arr, 0, -1)
        #print(f'x shape is: {x.shape}')

        prod = np.diagonal(x @ y, axis1=-1, axis2=-2)
        # need to normalize
        z = prod / np.sqrt(np.sum(x**2, axis=-1) * np.sum(y**2, axis=1))

        # store avg similarity across channels
        result[condition] = np.mean(z, axis=-1)

    return result

if __name__ == '__main__':

    from functools import partial

    path = '/home/matt/python/nri/scripting/threshold/data/psds.pkl'

    features = []

    # These two contribute a lot to the similarity and may be ignored
    #features.append(partial(area))
    #features.append(partial(max_height, prominence=2, width=0))

    features.append(partial(peak_counts, prominence=4, width=0))
    features.append(partial(max_prominence, prominence=4, width=0))
    features.append(partial(max_width, prominence=4, width=0))
    features.append(partial(max_location, prominence=4, width=0))

    featurized, names = featurize(path, 'ube3a', 'awake', features)

    m = similarity(featurized)


from collections import deque
from typing import List

class Solution:
    def optimizePairs(self):
        pairs=self.pairs
        starts = dict()
        ends = dict()
        nums = {}
        for i,(a, b) in enumerate(pairs):
            if a in starts:
                c,d=pairs[starts[a]]
                if d-c>a-b:
                    ends.pop(b)
                else:
                    continue
            if b in ends:
                c,d=pairs[ends[a]]
                if d-c>a-b:
                    starts.pop(a)
                else:
                    continue
            starts[a] = i
            ends[b] = i
            nums |= {a, b}
        active=[]
        lv=dict()
        return

    def findLongestChain(self, pairs: List[List[int]]) -> int:
        self.pairs = pairs
        starts = dict()
        ends = dict()
        nums = {-1001}
        for a, b in pairs:
            if starts.get(a, b) >= b:
                starts[a] = b
            if ends.get(b, a) <= a:
                ends[b] = a
            nums |= {a, b}
        nums = sorted(nums)
        cur_nums = deque()
        cur_best = dict()
        for e in nums:
            if e in starts:
                f = starts[e]
                v = 1
                for i in range(len(cur_nums)):
                    e2 = cur_nums[i]
                    if e2 >= e:
                        break
                    v = max(cur_best[e2] + 1, v)
                for i in range(len(cur_nums)):
                    if cur_nums[-1]<f:
                        break
                    v2=cur_best.pop(cur_nums.pop())
                    v=max(v2,v)
                cur_best[f] = v
                cur_nums.append(f)
            if e in ends:
                f = ends[e]
                while cur_nums[0] < f:
                    cur_best.pop(cur_nums.popleft())
        return max(cur_best.values())

def main():
    ex = [[1,2],[2,3],[3,4]]
    res = Solution().findLongestChain(ex)
    print(res)
    return

if __name__ == "__main__":
    main()

leetcode/daily/646 Maximum Length of Pair Chain.py


#!/usr/bin/env python3

import os
import sys
import csv
import re
import argparse
from unidecode import unidecode

# use as remove_non_ascii("...")
def remove_non_ascii(text):
    return unidecode(str(text))

def replace_trash(unicode_string):
     for i in range(0, len(unicode_string)):
         try:
             unicode_string[i].encode("ascii")
         except:
              #means it's non-ASCII
              unicode_string=unicode_string[i].replace(" ") #replacing it with a single space
     return unicode_string

def split(filehandler, delimiter=',', row_limit=10000,
          output_name_template='output_%s.csv', output_path='.', keep_headers=True,
          remove_all=False, zap_quotes_commas=False):
    """
    Splits a CSV file into multiple pieces.

    A quick bastardization of the Python CSV library.
    Arguments:
        `row_limit`: The number of rows you want in each output file. 10,000 by default.
        `output_name_template`: A %s-style template for the numbered output files.
        `output_path`: Where to stick the output files.
        `keep_headers`: Whether or not to print the headers in each output file.
    Example usage:

        >> from toolbox import csv_splitter;
        >> csv_splitter.split(open('/home/ben/input.csv', 'r'));

    """
    csv.field_size_limit(sys.maxsize)
    reader = csv.reader(filehandler, delimiter=delimiter)
    current_piece = 1
    current_out_path = os.path.join(
        output_path,
        output_name_template  % current_piece
    )
    current_out_writer = csv.writer(open(current_out_path, 'w'), dialect=csv.excel)#, quoting=csv.QUOTE_ALL)
    current_limit = row_limit
    real_line = 0
    if keep_headers:
        headers = next(reader)
        current_out_writer.writerow(headers)
    for i, row in enumerate(reader):
        #row = [s.replace('\0', "X") for s in row] # must be zapped prior
        #row = [re.sub(r'[\x00-\x1f\x7f-\x9f]', r' ', s) for s in row]
        row = [remove_non_ascii(s) for s in row]
        # keep LF, rm CR
        row = [re.sub(r'[\x00-\x09\x0b-\x1f\x7f-\x9f]', r' ', s) for s in row]
        # this used to be included by default...to handle genies
        if(zap_quotes_commas):
            row = [s.replace('"', ' ') for s in row]
            row = [s.replace(',', ' ') for s in row]
        #row = [replace_trash(s) for s in row]
        #row = [s.replaceAll("[^\\P{Cc}\\t\\r\\n]", "") for s in row]
        #print("{} - {}".format(i, row))
        if i + 1 > current_limit:
            current_piece += 1
            current_limit = row_limit * current_piece
            current_out_path = os.path.join(
                output_path,
                output_name_template  % current_piece
            )
            current_out_writer = csv.writer(open(current_out_path, 'w'), dialect=csv.excel)#, quoting=csv.QUOTE_ALL)
            if keep_headers:
                current_out_writer.writerow(headers)
        current_out_writer.writerow(row)

##
## split(open('/your/pat/input.csv', 'r'), row_limit=40000)
##
if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="CSV row cutter and cleaner")
    parser.add_argument("file", help="Input file.")
    parser.add_argument("--rows", type=int, help="Number of rows per chunk.", default=15000)
    parser.add_argument("--zap-quotes-commas", action='store_true',
                        help="Remove quotes and commas",
                        default=False)
    parser.add_argument("--remove-all", action='store_true',
                        help="Remove all bad chars from strings.", default=False)
    args = parser.parse_args()

    split(open(args.file), row_limit=args.rows,
          output_name_template=args.file + '_%s.csv',
          remove_all=args.remove_all,
          zap_quotes_commas=args.zap_quotes_commas)


#!/usr/bin/env python3
# coding=utf-8
"""Executa o servidor de nomes "uem.br"."""
import logging
import dns

def main():
    logging.basicConfig(
        format='[%(levelname)s]%(threadName)s %(message)s',
        level=logging.INFO)
    uemNS = dns.NameServer('uem.br', 3, '127.0.0.1', 10002)
    uemNS.add_record('deq.uem.br', '127.0.0.1:10003')
    uemNS.add_record('din.uem.br', '127.0.0.1:10004')
    uemNS.run()

if __name__ == '__main__':
    main()


from django.http import HttpResponse, HttpResponseRedirect
from django.template import RequestContext, loader
from django.http import Http404
from django.shortcuts import get_object_or_404, render
from django.utils import timezone
from .models import text

def index(request):
    latest_question_list = text.objects.order_by('-pub_date')[:50]
    print (latest_question_list)
    length = []
    for i in latest_question_list:
        length.append(len(i.question_text))

    context = {'latest_question_list': latest_question_list,
        'length': length,
    }
    if request.method == 'POST':
        # create a form instance and populate it with data from the request:
        q = text(title_text=request.POST['titill'], pub_date=timezone.now(),question_text=request.POST['txtArea'])
        q.save()
    return render(request, 'guestbook/index.html', context)

def detail(request, question_id):
    question = get_object_or_404(text, pk=question_id)
    context = {
        'question': question,
        'previous':int(question_id) - 1,
        'next':int(question_id) + 1,
    }
    return render(request, 'guestbook/detail.html', context)

def create(request):
    # if this is a POST request we need to process the form data

    return render(request, 'guestbook/create.html')

"""
def results(request, question_id):
    response = "You're looking at the results of question %s."
    return HttpResponse(response % question_id)

def vote(request, question_id):
    return HttpResponse("You're voting on question %s." % question_id)
"""


from typing import Dict, Optional
import tensorflow as tf

from orbit.launch import Task
from orbit.register import register_task_cls

from rocket.config import MaskedLanguageModelConfig
from rocket.common import registry
from rocket.model import BertPretrainerModel
from rocket.model.layers import nlp_layers

@register_task_cls(MaskedLanguageModelConfig)
class MaskedLanguageModelTask(Task):

  def _build_encoder(self, config):
    return registry.build_npl_encoder(config)

  def build_model(self):
    encoder_network = self._build_encoder(self.task_config.encoder)
    cls_heads = [
        nlp_layers.ClassificationHead(**config.as_dict())
        for config in self.task_config.cls_heads
    ] if self.task_config.cls_heads else []
    return BertPretrainerModel(
        mlm_activation=registry.get_activation(self.task_config.mlm_activation),
        mlm_initializer=tf.keras.initializers.TruncatedNormal(
            stddev=self.task_config.mlm_initializer_stddev),
        encoder_network=encoder_network,
        classification_heads=cls_heads,
    )

  def build_inputs(self, parameter, input_context: Optional[tf.distribute.InputContext] = None):
    if parameter.input_path == 'dummy':

      def dummy_data(_):
        dummy_ids = tf.zeros((1, parameter.sequence_length), dtype=tf.int32)
        dummy_lm = tf.zeros((1, parameter.max_predictions_per_seq), dtype=tf.int32)
        return dict(
            input_work_ids=dummy_ids,
            input_mask=dummy_ids,
            input_type_ids=dummy_ids,
            masked_lm_positions=dummy_lm,
            masked_lm_ids=dummy_lm,
            masked_lm_weights=tf.cast(dummy_lm, tf.float32),
            next_sentence_labels=tf.zeros((1, 1), dtype=tf.int32),
        )

      dataset = tf.data.Dataset.range(1)
      dataset = dataset.repeat()
      dataset = dataset.map(dummy_data, num_parallel_calls=tf.data.experimental.AUTOTUNE)
      return dataset

    return data_loader.get_data_loader(parameter).load(input_context)

  def build_losses(self, labels, model_outputs, aux_losses=None):
    losses = {}
    with tf.name_scope('MaskedLanguageModelTask/Losses'):
      lm_prediction_losses = tf.keras.losses.sparse_categorical_crossentropy(
          labels['masked_lm_ids'],
          tf.cast(model_outputs['mlm_logits'], tf.float32),
          from_logits=True)
      lm_label_weights = labels['masked_lm_weights']
      lm_numerator_loss = tf.reduce_sum(lm_prediction_losses * lm_label_weights)
      lm_denominator_loss = tf.reduce_sum(lm_label_weights)
      mlm_loss = tf.math.divide_no_nan(lm_numerator_loss, lm_denominator_loss)
      losses['losses/lm_example_loss'] = mlm_loss
      if 'next_sentence_labels' in labels:
        sentence_labels = labels['next_sentence_labels']
        sentence_outputs = tf.cast(
            model_outputs['next_sentence'], tf.float32)
        sentence_loss = tf.reduce_mean(
            tf.keras.losses.sparse_categorical_crossentropy(
                sentence_labels, sentence_outputs, from_logits=True))
        losses['losses/next_sentence_loss'] = sentence_loss

      total_loss = tf.add_n(list(losses.values()))
      if aux_losses:
        total_loss += tf.add_n(aux_losses)
    losses['losses/total'] = total_loss
    return losses

  def build_metrics(self, training=None):
    del training
    metrics = [
        tf.keras.metrics.SparseCategoricalAccuracy(name='mlm_accuracy'),
        tf.keras.metrics.Mean(name='mlm_example_loss'),
    ]
    if self.task_config.train_data.use_next_sentence_label:
      metrics.append(tf.keras.metrics.SparseCategoricalAccuracy(
          name='next_sentence_accuracy'))
      metrics.append(tf.keras.metrics.Mean(name='next_sentence_loss'))
    return metrics

  def process_metrics(self, metrics, lables, model_outputs, update_fns=None):
    del update_fns
    with tf.name_scope('MaskedLanguageModelTask/ProcessMetrics'):
      metrics = dict([(m.name, m) for m in metrics])
      if 'mlm_accuracy' in metrics:
        metrics['mlm_accuracy'].update_state(
            lables['masked_lm_ids'], model_outputs['mlm_logits'],
            lables['masked_lm_weights'])
        if 'next_sentence_accuracy' in metrics:
          metrics['next_sentence_accuracy'].update_state(
              lables['next_sentence_labels'],
              model_outputs['next_sentence'])

  def train_step(self, inputs, model: tf.keras.Model,
                 optimizer: tf.keras.optimizers.Optimizer,
                 metrics):
    with tf.GradientTape() as tape:
      outputs = model(inputs, training=True)
      losses = self.build_losses(
          labels=inputs,
          model_outputs=outputs,
          aux_losses=model.losses)
      total_loss = losses['losses/total']
      if self.task_config.scale_loss:
        scaled_loss = total_loss / tf.distribute.get_strategy().num_replicas_in_sync

    tvars = model.trainable_variables
    if self.task_config.scale_loss:
      grads = tape.gradient(scaled_loss, tvars)
    else:
      grads = tape.gradient(total_loss, tvars)
    optimizer.apply_gradients(list(zip(grads, tvars)))

    outputs.update(losses)
    self.process_metrics(metrics, inputs, outputs)
    return losses

  def validation_step(self, inputs, model: tf.keras.Model, metrics):
    outputs = model(inputs, training=False)
    losses = self.build_losses(
        labels=inputs,
        model_outputs=outputs,
        aux_losses=model.losses)
    self.process_metrics(metrics, inputs, outputs)
    return losses


import matplotlib.pyplot as plt
import numpy as np

def circle_imp(x0, y0, r):
    x = 0
    y = r
    pts_x = [x0+x, x0+x, x0+y, x0-y]
    pts_y = [y0+y, y0-y, y0+x, y0+x]
    d = 1 - r
    while y >= x:
        dE = 2*x + 1
        dSE = 2*x - 2*y + 1
        if d <= 0:
            d += dE
        else:
            d += dSE
            y = y - 1
        pts_x.append(x0+x)
        pts_x.append(x0-x)
        pts_x.append(x0+x)
        pts_x.append(x0-x)
        pts_x.append(x0+y)
        pts_x.append(x0-y)
        pts_x.append(x0+y)
        pts_x.append(x0-y)
        pts_y.append(y0+y)
        pts_y.append(y0+y)
        pts_y.append(y0-y)
        pts_y.append(y0-y)
        pts_y.append(y0+x)
        pts_y.append(y0+x)
        pts_y.append(y0-x)
        pts_y.append(y0-x)
        x += 1
    return pts_x, pts_y

def midpt_line(x0, y0, xn, yn):

    dy = yn-y0
    dx = xn-x0
    pts_x = []
    pts_y = []
    pts_x.append(x0)
    pts_y.append(y0)
    x = x0
    y = y0
    if dy >= 0 and dx >= 0:
        if dy > dx:
            d = 2*dx - dy
            dE = 2*dx
            dNE = 2*(dx-dy)
            for i in range(1, yn-y0 + 1):
                if d <= 0:
                    d = d + dE
                else:
                    d = d + dNE
                    x = x + 1
                pts_x.append(x)
                pts_y.append(y + i)
        else:
            d = 2*dy - dx
            dE = 2*dy
            dNE = 2*(dy-dx)
            for i in range(1, xn-x0+1):
                if d <= 0:
                    d = d + dE
                else:
                    d = d + dNE
                    y = y + 1
                pts_x.append(x + i)
                pts_y.append(y)
    else:
        if abs(dy) > abs(dx):
            dy = abs(dy)
            dx = abs(dx)
            d = 2*dx - dy
            dE = 2*dx
            dNE = 2*(dx-dy)
            for i in range(1, abs(yn-y0)+1):
                if d <= 0:
                    d = d + dE
                else:
                    d = d + dNE
                    x = x + 1
                pts_x.append(x)
                pts_y.append(y - i)
        else:
            dy = abs(dy)
            dx = abs(dx)
            d = 2*dy - dx
            dE = 2*dy
            dNE = 2*(dy-dx)
            for i in range(1, abs(xn-x0)+1):
                if d <= 0:
                    d = d + dE
                else:
                    d = d + dNE
                    y = y - 1
                pts_x.append(x + i)
                pts_y.append(y)
    return pts_x, pts_y

def FloodFill(screen, x, y, newC, oldC):

    currPix = screen[y, x]
    if currPix == oldC:
        screen[y, x] = newC
        FloodFill(screen, x + 1, y, newC, oldC)
        FloodFill(screen, x - 1, y, newC, oldC)
        FloodFill(screen, x, y + 1, newC, oldC)
        FloodFill(screen, x, y - 1, newC, oldC)

    x1 = np.zeros((100, 100))
    coor_y, coor_x = midpt_line(50, 40, 80, 40)
    for i, j in zip(coor_x, coor_y):
        x1[i, j] = 255
    coor_y, coor_x = midpt_line(50, 40, 50, 80)
    for i, j in zip(coor_x, coor_y):
        x1[i, j] = 255
    coor_y, coor_x = midpt_line(50, 80, 80, 80)
    for i, j in zip(coor_x, coor_y):
        x1[i, j] = 255
    coor_y, coor_x = midpt_line(80, 80, 80, 40)
    for i, j in zip(coor_x, coor_y):
        x1[i, j] = 255
    FloodFill(x1, 60, 50, 100, 0)
    plt.figure()
    plt.imshow(x1)
    plt.gca().invert_yaxis()
    obj1 = np.where(x1 > 0)

    plt.figure()
    x2 = np.zeros((100, 100))
    coor_y, coor_x = midpt_line(10, 10, 10, 50)
    for i, j in zip(coor_x, coor_y):
        x2[i, j] = 255
    coor_y, coor_x = midpt_line(10, 50, 30, 50)
    for i, j in zip(coor_x, coor_y):
        x2[i, j] = 255
    coor_y, coor_x = midpt_line(30, 50, 30, 10)
    for i, j in zip(coor_x, coor_y):
        x2[i, j] = 255
    coor_y, coor_x = midpt_line(10, 10, 30, 10)
    for i, j in zip(coor_x, coor_y):
        x2[i, j] = 255
    FloodFill(x2, 20, 20, 50, 0)
    plt.figure()
    plt.imshow(x2)
    plt.gca().invert_yaxis()
    obj2 = np.where(x2 > 0)

    plt.figure()
    x3 = np.zeros((100, 100))
    coor = []
    coor_y, coor_x = circle_imp(50, 30, 30)
    for i, j in zip(coor_x, coor_y):
        x3[i, j] = 255
        coor.append((j, i))
    FloodFill(x3, 50, 20, 200, 0)
    plt.figure()
    plt.imshow(x3)
    plt.title('SCAN FILL METHOD')
    plt.gca().invert_yaxis()
    obj3 = np.where(x3 > 0)

    plt.figure()
    x4 = np.zeros((100, 100))
    coor = []
    coor_y, coor_x = circle_imp(60, 60, 25)
    for i, j in zip(coor_x, coor_y):
        x4[i, j] = 255
        coor.append((j, i))
    FloodFill(x4, 60, 60, 150, 0)
    plt.figure()
    plt.imshow(x4)
    plt.gca().invert_yaxis()
    obj4 = np.where(x4 > 0)

    x5 = np.zeros((100, 100))
    coor_y, coor_x = midpt_line(70, 10, 90, 10)
    for i, j in zip(coor_x, coor_y):
        x5[i, j] = 255
    coor_y, coor_x = midpt_line(90, 10, 90, 30)
    for i, j in zip(coor_x, coor_y):
        x5[i, j] = 255
    coor_y, coor_x = midpt_line(70, 30, 90, 30)
    for i, j in zip(coor_x, coor_y):
        x5[i, j] = 255
    coor_y, coor_x = midpt_line(70, 10, 70, 30)
    for i, j in zip(coor_x, coor_y):
        x5[i, j] = 255
    FloodFill(x5, 80, 20, 250, 0)
    plt.figure()
    plt.imshow(x5)
    plt.gca().invert_yaxis()
    obj5 = np.where(x5 > 0)

    depth = np.full((100, 100), 10000)
    new = np.zeros((100, 100))

    for poly in [(obj1, 100, 5), (obj2, 50, 2), (obj3, 200, 10), (obj4, 150, 20), (obj5, 250, 1)]:
        d = poly[2]
        c = poly[1]
        poly = poly[0]
        for i, j in zip(poly[0], poly[1]):
            if depth[i, j] > d:
                depth[i, j] = d
                new[i, j] = c
    plt.figure()
    plt.imshow(new)
    plt.gca().invert_yaxis()


#! -*- coding: utf-8 -*-
#装备查询

import sys
import sys
import time
import json
import re
import requests
import threading

from src.base_spider import BaseSpider
from src.util import MyLogger
import src.request_util
from src.db import MysqlPool
from src.db import equip_upsert

PAGE_MAX = 100
PRICE_MAX = 1000000

equip_log = MyLogger().new('equip.log')

class EquipSpider(BaseSpider):
    def __init__(self, kind_id, sql_conn, query_url=r'https://recommd.xyq.cbg.163.com/cgi-bin/recommend.py'):
        super(EquipSpider, self).__init__()
        self.kind_id = kind_id
        self.callback_index = 1
        self.proxy_fetcher = src.request_util.get_proxy_fetcher()
        self.query_url = query_url
        self.query_succ = 0
        self.query_fail = 0
        self.conn = sql_conn

    def get_query_params(self, price_min, price_max, page):
        return {
            'callback': 'Request.JSONP.request_map.request_%d' % self.callback_index,
            'act': 'recommd_by_role',
            'page':page,
            'level_min': 90,
            'level_max': 160,
            'price_min': price_min * 100,
            'price_max': price_max * 100,
            'kindid': self.kind_id,
            'sum_attr_without_melt': 1,
            'count': 15,
            'search_type': 'overall_search_equip',
            'view_loc': 'overall_search'
        }

    @staticmethod
    def get_producer(desc):
        desc = re.sub(r"#Y", "|", desc)
        desc = re.sub(r"#[a-zA-Z]|[A-Z0-9]", "", desc)
        desc = re.sub(r" ", "", desc)
        desc = re.sub(r"强化打造", "", desc)
        lists = desc.split("|")
        for _k in lists:
            if "制造者" in _k:
                name = _k.split("：")[-1]
                return name

    def get_session(self):
        sess = requests.Session()
        header = src.request_util.get_header()
        header['Referer'] = 'https://xyq.cbg.163.com/cgi-bin/equipquery.py?act=show_overall_search_equip'
        sess.headers.update(header)
        return sess

    def get_proxies(self):
        proxies = {}
        proxy_list = self.proxy_fetcher.get_proxies()
        if len(proxies) < 2: return proxies
        proxies['http'] = proxy_list[0]
        proxies['https'] = proxies[1]
        return proxies

    @staticmethod
    def check_status_code(status_code):
        """
        藏宝阁自己的状态码，如果被屏蔽回返回不同的状态码,暂时不处理
        TODO: 处理验证码情况
        """
        equip_log.debug("Status code return. code=%d", status_code)
        if status_code != 1:
            return False
        return True

    def parse_and_save(self, data):
        """
        @param
        data: json格式的原始数据
        return: 返回[{}], 包含了装备数据
        """
        #当查询范围过小时，equips可能为空，这是正常情况
        equips_list = data['equips']
        if not equips_list: return
        #解析出制造者，然后打包插入数据库
        upsert_list = []
        for equip in equips_list:
            fixed_data = {}
            other_info = equip['other_info']
            desc = json.loads(other_info)['desc']
            producer = self.get_producer(desc)
            fixed_data['producer'] = producer
            fixed_data['brief_desc'] = desc
            fixed_data['price_int'] = equip['price_int']
            fixed_data['equip_name'] = equip['equip_name']
            fixed_data['server_name'] = equip['server_name']
            fixed_data['equip_level'] = equip['equip_level']
            fixed_data['info_dumps'] = json.dumps(equip)
            fixed_data['game_ordersn'] = equip['game_ordersn']
            upsert_list.append(fixed_data)

        equip_upsert(self.conn, upsert_list)

    def query_single_page(self, sess, page, price_min, price_max):
        equip_log.info("Request page: price_min=%d,price_max=%d,page=%d", price_min, price_max, page)
        params = self.get_query_params(price_min, price_max, page)
        try:
            proxies = self.get_proxies()
            response = sess.get(self.query_url, params=params, proxies=proxies)
        except Exception as e:
            print("Request error:", e)
            time.sleep(5)
            return None, ""
        if response.status_code != requests.codes.ok:
            equip_log.error("Request status code=%d", response.status_code)
            return None, response.text
        page_data = src.request_util.loads_jsonp(response.text)
        if not page_data:
            equip_log.error("loads jsonp error:%s", response.text)
            return None, response.text
        if not self.check_status_code(page_data['status']): return None, response.text
        self.callback_index = self.callback_index + 1
        return page_data, response.text

    def query_by_range(self, sess, price_min, price_max):
        """
        按价格返回查询装备数据
        @return boolean, price_max -- 第一个值表示是否正常查询，第二个值表示此次查询所用的price_max
        """
        cur_page = 1
        #先取一页确定价格范围
        page1_data, text = self.query_single_page(sess, cur_page, price_min, price_max)
        if not page1_data:
            self.query_fail = self.query_fail + 1
            equip_log.error("Request page data error. text=%s", text)
            return False, price_max
        self.query_succ = self.query_succ + 1
        if not page1_data['equips']: return True, price_max
        total_page = page1_data["pager"]["total_pages"]
        if total_page >= PAGE_MAX:
            price_max = price_min + (price_max - price_min) / 2
            return True, price_max

        for i in range(cur_page, total_page+1):
            page_data, text = self.query_single_page(sess, i, price_min, price_max)
            if not page_data:
                equip_log.error('Request single page failed. index=%d,price_min=%d,price_max=%d',
                                    i, price_min, price_max)
                continue
            self.parse_and_save(page_data)
        return True, price_max

    def run(self):
        price_min = 300
        price_max = 301
        range = 10
        sess = self.get_session()
        while True:
            ret, cur_price_max = self.query_by_range(sess, price_min, price_max)
            equip_log.info('Request stat. succ=%d,fail=%d', self.query_succ, self.query_fail)
            if not ret:
                time.sleep(1)
                equip_log.debug('Request range failed.')
                continue
            if cur_price_max >= PRICE_MAX: return

            #价格范围太大，缩小返回重新查询
            if cur_price_max < price_max:
                range = range / 2
            else:
                range = range * 2
            price_min = cur_price_max + 1
            price_max = min(price_min + range, PRICE_MAX)