 # Path: spam_detect_app/app.py

import tensorflow_hub as hub
import tensorflow_text as text
import tensorflow as tf
from sklearn.model_selection import train_test_split
import pickle
import pandas as pd
import numpy as np

EPOCHS = 3

def embed_text_bert(spam_data):
    '''returns the pooled output of embedded input'''

    # bert modle not running due to missing files try to fix later,/
    # im going to embed data myself, using
    # text vectotizer

    #bert api
    pre_process_url = 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3'
    encoder_url = 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/4'

    bert_preprocess_model = hub.KerasLayer(pre_process_url)
    bert_model = hub.KerasLayer(encoder_url)

    preprocessed_text = bert_preprocess_model(spam_data)

    print ('...embedding data')

    embedding_result = bert_model(preprocessed_text)
    print('data embedding complete')

    return embedding_result['pooled_output']

def load_data():
    '''returns the data as a pandas dataframe'''
    print ('...loading data')
    spam_data = pd.read_csv('spam_detect_app/spam_preprocessed.csv')#development location is spam_detect_app/spam_preprocessed.csv
    print('data load successfull')
    return spam_data

def embed_text(spam_data):
    '''adapts data and returns textVectorizer model with the adapted data'''
    textVectorizer = tf.keras.layers.experimental.preprocessing.TextVectorization(max_tokens=100000, output_sequence_length=200)

    textVectorizer.adapt(spam_data) #adapt data to create vocabulary

    return textVectorizer

def vectorize_text(textVectorizer, spam_data):
    '''returns input text as an embedded text in numpy'''

    embedded_text = textVectorizer(spam_data) #vectorize data

    return np.array(embedded_text)

def pipe_data(x_data,y_data):
    '''returns a piped dataset(cached,shuffled,batched and prefetched) from the given data as a tuple of tensors'''
    test_data = tf.data.Dataset.from_tensor_slices((x_data,y_data))
    test_data = test_data.cache()
    test_data = test_data.shuffle(6000)
    test_data = test_data.batch(15)
    test_data = test_data.prefetch(5)

    return test_data

def model_create():
    '''returns a model with the given parameters'''
    input = tf.keras.layers.Input(shape=(200,))# Tensorflow autograph can race LSTM in a function, consider putting it in the main execution block
    embedding_layer = tf.keras.layers.Embedding(100000, 128)(input)
    lstm_layer = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32, activation='tanh', return_sequences=True))(embedding_layer)
    flatten = tf.keras.layers.Flatten()(lstm_layer)
    hidden_layer1 = tf.keras.layers.Dense(32,activation= 'relu', kernel_regularizer=tf.keras.regularizers.l1(0.0001))(flatten)
    hidden_drop1 = tf.keras.layers.Dropout(0.2)(hidden_layer1)
    hidden_layer2 = tf.keras.layers.Dense(64,activation= 'relu',kernel_regularizer=tf.keras.regularizers.l1(0.001))(hidden_drop1)
    hidden_drop2 = tf.keras.layers.Dropout(0.5)(hidden_layer2)
    hidden_layer3 = tf.keras.layers.Dense(32,activation= 'relu', kernel_regularizer=tf.keras.regularizers.l1(0.0001))(hidden_drop2)
    hidden_drop3 = tf.keras.layers.Dropout(0.2)(hidden_layer3)
    output = tf.keras.layers.Dense(1,activation= 'sigmoid')(hidden_drop3)

    model = tf.keras.models.Model(inputs = input, outputs = output)
    model.compile(optimizer='adam',loss='binary_crossentropy',metrics=['accuracy'])

    return model

if __name__ == '__main__':
    data = load_data()

    vectorizer = embed_text(data.Message)
    vectorized_data = vectorize_text(vectorizer,data.Message)

    ##split data into train and test samples
    x_train, x_test, y_train, y_test= train_test_split(vectorized_data,
                                                         data.spam,
                                                        test_size=0.1)

    #data pipeline
    test_data = pipe_data(x_train,np.expand_dims(y_train, axis=1))

    print(f'np.expand_dims(y_train, axis=1).shape:{np.expand_dims(y_train, axis=1).shape}')

    print(f'//////////////////////////TRAIN DATA///////////')

    model  = model_create()
    model.fit(test_data,epochs = EPOCHS)
    print('train done')

    print(f'//////////////////////////saving model///////////')
    tf.keras.models.save_model(model, 'trained_model')

    print(f'//////////////////////////saving model done///////////')

    print(f'//////////////////////////Saving Test data///////////')
    with open('test_data', 'wb') as f:#development location is spam_detect_app/test_data
        pickle.dump(x_test, f)

    with open('spam_col_test_data', 'wb') as f:#development location is spam_detect_app/spam_col_test_data
        pickle.dump(y_test,f)

    print(f'//////////////////////////Saving Test data done///////////')

    print(f'//////////////////////////Loading Trained model///////////')
    trained_model  = tf.keras.models.load_model('trained_model')

    print(f'//////////////////////////Loading Test Data///////////')
    with open('test_data', 'rb') as f:#development location is spam_detect_app/test_data
        x_test = pickle.load(f)
        print ('...test data load successfull')

    with open('spam_col_test_data', 'rb') as f:#development location is spam_detect_app/spam_col_test_data
        y_test = pickle.load(f)
        print ('...spam_col_test_data successfull')

#\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\test below
    # vectorized_data = vectorize_text(vectorizer,"WINNER!! As a valued network customer you have been selected to receivea £900 prize reward! To claim call 09061701461. Claim code KL341. Valid 12 hours only.")
    # print(f'vectorized_data.shape:{vectorized_data.shape}')

    # print(f'np.expand_dims(vectorized_data, axis=0).shape: {np.expand_dims(vectorized_data, axis=0).shape}' )
    # y_hat = model.predict(np.expand_dims(vectorized_data, axis=0))

    # print(y_hat)
    # print(f"y_hat.shape: {y_hat.shape}")
    # # /print(f"y_hat[0][0]: {y_hat[0][0]}")

    # y_hat = predict(x_test)
#\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\test above

    y_hat = model.predict(x_test)

    y_true = np.array(y_test)
    y_hat = np.array(y_hat)

    # print (y_true)
    print(y_hat.shape)
    print(y_true.shape)

    print(np.array(y_hat))
    print(y_true)


#!/usr/bin/python
# -*- coding: utf-8 -*-
# Imports
import argparse
import os

from noaaplotter.download_utils import download_era5_from_gee

def main():
    """
    Main Function
    :return:
    """
    ##### Parse arguments #####
    parser = argparse.ArgumentParser(description='Parse arguments.')

    parser.add_argument('-o', dest='output_file', type=str, required=True,
                        default='data/data.csv',
                        help='csv file to save results')

    parser.add_argument('-lat', dest='lat', type=float, required=True,
                        help='Latitude of selected location')

    parser.add_argument('-lon', dest='lon', type=float, required=True,
                        help='Longitude of selected location')

    parser.add_argument('-loc', dest='loc_name', type=str, required=False,
                        default='',
                        help='Location name')

    parser.add_argument('-dt', dest='datatypes', type=list, required=False, default=['TMIN', 'TMAX', 'PRCP', 'SNOW'])

    parser.add_argument('-start', dest='start_date', type=str, required=True,
                        help='start date of plot ("yyyy-mm-dd")')

    parser.add_argument('-end', dest='end_date', type=str, required=True,
                        help='end date of plot ("yyyy-mm-dd")')

    args = parser.parse_args()

    # remove file if exists
    if os.path.exists(args.output_file):
        os.remove(args.output_file)

    download_era5_from_gee(latitude=args.lat,
                           longitude = args.lon,
                           end_date= args.end_date,
                           start_date = args.start_date,
                           output_file = args.output_file)

if __name__ == "__main__":
    main()


from datetime import datetime

import click
import prejus

def validate_options(ctx, param, value):
    if not value:
        return param.default
    try:
        classname = param.name.title()
        cls = getattr(prejus, classname)
        return cls[value]
    except KeyError as error:
        raise click.BadParameter('"{}" não é uma opção válida.'.format(value))

def validate_dates(ctx, param, value):
    if not value:
        return param.default
    try:
        return datetime.strptime(value, '%d-%m-%Y').date()
    except (TypeError, ValueError) as error:
        raise click.BadParameter(
            '"{}" não é um valor válido. Use "dd-mm-aaaa".'.format(value))

def validate_orgao(ctx, param, value):
    try:
        orgao = ('orgaoSuperior', prejus.OrgaoSuperior[value])
    except KeyError:
        try:
            orgao = ('unidade', prejus.Unidade[value])
        except KeyError:
            raise click.BadParameter(
                '"{}" não é um valor válido para {}.'.format(value, param))
    return orgao

def csvify(listrow):
    return ','.join([
        ('"{}"' if ' ' in str(col) else '{}').format(col) for col in listrow
    ])

@click.command()
@click.option('--inicio', callback=validate_dates,
              help='Data de início da consulta.')
@click.option('--fim', callback=validate_dates,
              help='Data de fim da consulta.')
@click.option('--elemento', callback=validate_options,
              help='Elemento de despesa.')
@click.option('--fase', callback=validate_options, default='EMPENHO',
              help='Fase da despesa.')
@click.argument('orgao', callback=validate_orgao)
def cli(**kwargs):
    '''
    Consulta despesas do Judiciário Brasileiro, no Portal da Transparência.
    '''
    orgao = kwargs.pop('orgao')
    kwargs[orgao[0]] = orgao[1]
    kwargs = {key: value for key, value in kwargs.items() if value}
    results = prejus.consulta(**kwargs)

    if results:
        click.echo(csvify(prejus.HEADERS))
        for row in results:
            click.echo(csvify(row.values()))


from datetime import datetime
from rabbitpy import Connection, Message
from rabbitpy.exceptions import MessageReturnedException

import sys
import os

sys.path.append(
    os.path.dirname(
        os.path.abspath(
            os.path.dirname(os.path.abspath(os.path.dirname(__file__)))
        )
    )
)

from config.config_rabbitpy import URL, EXCHANGE

body = "routing failure test"
properties = {
    "content_type": "text/plain",
    "timestamp": datetime.now(),
    "message_type": "routing failure",
}

with Connection(URL) as conn:
    try:
        with conn.channel() as channel:
            msg = Message(
                channel=channel, body_value=body, properties=properties
            )

            # * Exchange와 Routing Key에 바인딩한 Queue가 없기 때문에 라우팅할 수 없음
            # * Exception은 Channel 수준에서 난다!!
            is_published = msg.publish(
                exchange=EXCHANGE,
                routing_key="routing.failure.test",
                mandatory=True,
            )
            print(is_published)  # None
    except MessageReturnedException as e:
        print(f"Publish failed: {e}")
        # * Message was returned by RabbitMQ: (312) for exchange NO_ROUTE

"""
rabbitpy 라이브러리에서 클라이언트는 Basic.Return을 자동으로 수신하며,
채널 범위에서 수신하면 MessageReturnedException을 발생시킴.

다른 라이브러리의 경우,
메시지를 발행할 때 RabbitMQ에서 Basic.Return RPC를 전달받으면 실행할 콜백 메소드를 등록해야 할 수도 있음.
비동기적으로 Basic.Return 메시지를 처리할 때 다른 메시지를 소비하는 것처럼
Basic.Return 메소드 프레임, 콘텐츠 헤더 프레임, 바디 프레임을 받게 됨.
"""


import os
import torch
# End2end inversion of ours, Runing for pix2pix-zero datasets. 18 images (cat, dog)

SCALE_FACTOR_2=0.5
idx = 0
# 打开文本文件以读取模式
with open('prompt_all_imgs.txt', 'r') as file:
    # 逐行读取文件内容
    for line in file:
        idx += 1
        # 使用分号分隔每一行
        input_img, longprompt = line.strip().split(': ')
        if idx < 50:
            prompt = input_img.split('/')[-2]
        else:
            prompt = input_img.split('/')[-1].split('.')[0].split('_')[0]
        prompt = 'a ' + prompt

        result = os.system(f"CUDA_VISIBLE_DEVICES=7  python end2end_inv_stage2.py  \
                            --input_image '{input_img}' \
                            --prompt_str '{prompt}' \
                            --long_prompt_str '{longprompt}'    \
                            --results_folder 'output/all_imgs_inversion_in_prompt_file' \
                            --scale_factor_2 {SCALE_FACTOR_2}   \
                            ")
        if result == 0:
            print("Command executed successfully.")
        else:
            print("Command failed.")
            torch.cuda.empty_cache()


"""App model tests.
"""
from unittest import mock

import pytest
from django.db.models import signals

from .. import models
from ..exceptions import CameraRtspInvalid
from ..models import Camera
from .factories import CameraFactory

pytestmark = pytest.mark.django_db

# pylint: disable=W0613
# skip unused-argument mock_cv2_capture

@pytest.mark.fast
def test_rtsp(mock_cv2_capture):
    """test_rtsp.

    Camera should save uppercase rtsp while using lowercase for cv2
    """
    cam1 = CameraFactory()
    cam1.rtsp = "RTSP://fake_url"
    cam1.save()
    assert cam1.rtsp == "RTSP://fake_url"

@pytest.mark.fast
def test_rtsp_none(mock_cv2_capture):
    """test_rtsp."""
    cam1 = CameraFactory()
    cam1.rtsp = None
    with pytest.raises(CameraRtspInvalid):
        cam1.save()

@pytest.mark.fast
def test_verify_fail(monkeypatch):
    """test_rtsp."""
    # verify fail
    monkeypatch.setattr(models, "verify_rtsp", mock.MagicMock(return_value=False))
    with pytest.raises(CameraRtspInvalid):
        cam1 = CameraFactory()
        cam1.save()

@pytest.mark.fast
def test_model_str_method(mock_cv2_capture):
    """test_rtsp."""

    cam_obj = CameraFactory()
    assert str(cam_obj) == cam_obj.name

@pytest.mark.fast
def test_model_repr_method(mock_cv2_capture):
    """test_rtsp."""

    cam_obj = CameraFactory()
    assert cam_obj.__repr__() == cam_obj.name.__repr__()

@pytest.mark.fast
def test_model_skip_signals():
    """test_rtsp."""
    signals.pre_save.disconnect(sender=Camera, dispatch_uid="Camera_pre")
    cam_obj = CameraFactory()
    cam_obj.is_demo = False
    cam_obj.save()
    signals.pre_save.connect(Camera.pre_save, Camera, dispatch_uid="Camera_pre")

    with pytest.raises(CameraRtspInvalid):
        cam_obj.save()
    cam_obj.skip_signals = True
    cam_obj.save()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

""" ASCII rendering module for flying donut.

For more information, see README.

For usage, run <python3 donut.py>.

Project can be found here <https://github.com/ingranys/flying-donut>.

This program is free software: you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation, either version 3 of the License, or (at your option) any later
version.

This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.

You should have received a copy of the GNU General Public License along with
this program. If not, see <http://www.gnu.org/licenses/>.
"""

__author__ = "ingranys"
__contact__ = "ingranys@protonmail.com"
__copyright__ = "Copyright 2021, Mustapha Gaies, Toulouse (France)"
__date__ = "2021/01/16"
__deprecated__ = False
__email__ =  "ingranys@protonmail.com"
__license__ = "GPLv3"
__maintainer__ = None
__status__ = "Production"
__version__ = "0.1.0"

import sys
import time
import curses
import numpy

def warning(mode='<mode>',duration=10):
    """Print user warning.

    Args:
        mode (str, optional): Mode (can PREVIEW or DEBUG). Defaults to '<mode>'.
        duration (int, optional): Message duration in seconds. Defaults to 10.
    """
    print('WARNING!')
    print('{0} mode is enabled with a large number of points.'.format(mode))
    print('Be aware of possible lag while rendering.')
    for remaining in range(duration, -1, -1):
        print("Continue in {:2d} seconds...".format(remaining),end="\r")
        time.sleep(1)
    # Use ASCII escape sequence
    #https://gist.github.com/fnky/458719343aabd01cfb17a3a4f7296797
    print("\033[2K",end="\r")
    print("Moving on to the rendering.")

def screen(n_pixels):
    """Initialize ascii screen.

    Args:
        n_pixels (int): Screen size in pixels (each pixel is ASCII character).

    Returns:
        scr (screen): Initialized screen.
        frame_height (int) : ASCII frame height.
        frame_width (int) : ASCII frame width (ASCII image aspect ratio is 2:1).
    """
    # Initialize screen
    scr = curses.initscr()

    # Get the maximum possible size for the current console
    screen_height,screen_width= scr.getmaxyx()
    # Get the maximum possible size for the ascii screen within the console
    # IMPORTANT! It takes twice as much columns than lines to draw ascii art
    frame_size = min(screen_height,numpy.floor(screen_width/2).astype(int))
    if n_pixels<=0:
        # Auto-scale
        frame_height = frame_size
        frame_width = 2*frame_size
    elif n_pixels<=frame_size:
        # Manual scale
        frame_height = n_pixels
        frame_width = 2*n_pixels
    else :
        # Contraints can't be applied
        # Go back to previous console display and print error message
        curses.endwin()
        print('ERROR!')
        print('Image size exceeds console size.')
        print('Please decrease pixels number or widen console.')
        sys.exit()

    # Clear screen to finish initialization
    scr.clear()
    scr.refresh()

    return scr,frame_height,frame_width

def reset():
    """Go back to previous console display
    """
    curses.endwin()

def asciis(pixels,char):
    """Convert 2D grayscale image to ASCII characters.

    Args:
        pixels (array(float)): 2D array representing graysclale image.
        char (array(str)): List of ASCII characters.

    Returns:
        ascii_characters (array(str)):
    """
    n_rows, n_columns = pixels.shape
    n_char = len(char)
    # Map intensity values to characters
    ascii_characters =   [
                            [char[int(numpy.floor(n_char*pixels[i,j]))]
                            for j in range(n_columns)]
                        for i in range(n_rows)]

    return ascii_characters

def render(ascii_characters,screen,current_frame,n_frames):
    """Print ASCII characters to console.

    Args:
        ascii_characters (array[str]): 2D array containing ASCII characters.
        screen (screen): ASCII screen.
        current_frame (int): Current frame number.
        n_frames (int): Total number of frames.
    """
    # Get frame shape
    n_rows = len(ascii_characters)
    n_columns = len(ascii_characters[0])
    # Print ASCII characters one by one
    for i in range(n_rows):
        for j in range(n_columns):
            screen.addstr(i, j, ascii_characters[i][j])
    msg = '{0}/{1}'.format(current_frame+1,n_frames)
    # Refresh screen
    screen.addstr(0, 0, msg )
    screen.refresh()


import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import classification_report
import tensorflow as tf
from tensorflow.keras import layers

file_path = 'training_data.txt'
df = pd.read_csv(file_path, sep='\s+', engine='python')
outpur_path = 'test_data.txt'
df1 = pd.read_csv(outpur_path, sep='\s+', engine='python')

# Change activity labels 7, 8, 9, 10, 11, and 12 to 7
df['activity'] = df['activity'].replace({8: 7, 9: 7, 10: 7, 11: 7, 12: 7})

# Separate features (X) and labels (y)
X = df.drop('activity', axis=1)
y = df['activity']

# Fit the scaler on the training data (X)
scaler = StandardScaler().fit(X)

# Scale the training data
X_scaled = scaler.transform(X)
df1_scaled = scaler.transform(df1)

# Perform train-test split
X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2, random_state=42)

# One-hot encode the labels
y_train_onehot = tf.keras.utils.to_categorical(y_train - 1)
y_test_onehot = tf.keras.utils.to_categorical(y_test - 1)

# Create a simple MLP model
model = tf.keras.Sequential([
    layers.Input(shape=(X_train.shape[1],)),
    layers.Dense(256, activation='relu'),
    layers.Dropout(0.2),
    layers.Dense(128, activation='relu'),
    layers.Dropout(0.2),
    layers.Dense(y_train_onehot.shape[1], activation='softmax')
])

# Compile the model
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])

# Train the model
model.fit(X_train, y_train_onehot, batch_size=32, epochs=10, validation_split=0.1)

# Evaluate the model
y_pred_onehot = model.predict(X_test)
y_pred = y_pred_onehot.argmax(axis=1) + 1

# Print the performance metrics
print(classification_report(y_test, y_pred))
# Perform predictions on df1
df1_pred_onehot = model.predict(df1_scaled)
df1_pred = df1_pred_onehot.argmax(axis=1) + 1

# Save the predictions to a txt file
with open('predictions1.txt', 'w') as f:
    for prediction in df1_pred:
        f.write(f"{prediction}\n")


# python3.7
# -*- coding: utf-8 -*-
# @Project_Name  : Air Quality prediction Beijing
# @File          : save_model.py
# @Time          : 2022/11/26 14:40
# @Author        : SY.M
# @Software      : PyCharm
import torch
import os

def save_model(model_name: str, target_feature: str, result: tuple, state_dict: dict, pred_len: int, seq_len: int,
        root='G:\PyCharmWorkSpace\PyCharmProjects\Air quality\Air Quality prediction Beijing\saved_model',):
    save_path = os.path.join(root, model_name)
    if not os.path.exists(save_path):
        os.makedirs(save_path)
    save_file = os.path.join(save_path, f'{model_name} {target_feature} {seq_len}->{pred_len} {result}.pt')
    torch.save(state_dict, save_file)
    print('save model successful！')

def load_model(file_path):
    torch.load(file_path)


"""
Clean up software environments created by `coiled-bisect.py`, aka all environments that start with a given prefix.
"""
import asyncio
import sys
import coiled

ACCOUNT = "dask-engineering"

async def delete_senvs(prefix: str):
    async with coiled.Cloud(asynchronous=True) as client:
        senvs = await client.list_software_environments(ACCOUNT)
        matches = [s for s in senvs if s.split("/")[1].startswith(prefix)]
        input(f"Going to delete {matches}. Press enter to confirm, Ctrl-C to exit: ")

        await asyncio.gather(
            *(
                client.delete_software_environment(senv, account=ACCOUNT)
                for senv in matches
            )
        )

if __name__ == "__main__":
    if len(sys.argv) != 2:
        print(
            "Pass the software environment prefix as the only CLI argument, like `python cleanup.py senv-foo`"
        )
        sys.exit(1)

    asyncio.run(delete_senvs(sys.argv[1]))


from PyQt6.QtWidgets import (
    QApplication,
    QFileDialog,
    QInputDialog,
    QMainWindow,
    QWidget,
    QBoxLayout,
    QDockWidget,
    QLineEdit,
    QLabel,
    QCheckBox,
    QPushButton,
    QGridLayout,
    QFrame,
    QDoubleSpinBox,
    QSpinBox,
    QDialog,
    QGraphicsView,
    QGraphicsScene,
    QToolBar,
    QSizePolicy,
)
from PyQt6.QtCore import (
    Qt,
    QTimer,
    QEvent,
)
from PyQt6 import QtGui
from pytube import YouTube
import librosa
import os
from . import audio_view, events
from ..audio import audio
from ..utils import utils

MUSIC_PATH = "music"
ICONS_PATH = "icons"

def icon(path):
    return QtGui.QIcon("icons/{}".format(path))

class VerticalPitchTrackingWidget(QWidget):
    def __init__(self, audio_player, *args, **kargs):
        super(VerticalPitchTrackingWidget, self).__init__(*args, **kargs)

        self.audio_player = audio_player

        main_layout = QBoxLayout(QBoxLayout.Direction.TopToBottom)
        controls_layout = QBoxLayout(QBoxLayout.Direction.LeftToRight)

        self.timestamp_input = QLineEdit("00.00")
        self.timestamp_input.returnPressed.connect(self.on_set_timestamp)
        controls_layout.addWidget(self.timestamp_input)

        self.lock_to_timestamp_checkbox = QCheckBox("Lock to playback timestamp")
        self.lock_to_timestamp_checkbox.setChecked(True)
        controls_layout.addWidget(self.lock_to_timestamp_checkbox)

        self.threshold_input_label = QLabel("Threshold:")
        controls_layout.addWidget(self.threshold_input_label)
        self.threshold_input = QDoubleSpinBox()
        self.threshold_input.setValue(0.1)
        self.threshold_input.setMaximum(1.0)
        self.threshold_input.setMinimum(0.0)
        self.threshold_input.setSingleStep(0.1)
        controls_layout.addWidget(self.threshold_input)

        self.window_width_label = QLabel("Window size:")
        controls_layout.addWidget(self.window_width_label)
        self.window_width_input = QSpinBox()
        self.window_width_input.setValue(2048)
        self.window_width_input.setMinimum(1024)
        self.window_width_input.setMaximum(10000)
        controls_layout.addWidget(self.window_width_input)

        refresh_button = QPushButton("Refresh")
        refresh_button.clicked.connect(self.on_refresh)
        controls_layout.addWidget(refresh_button)

        main_layout.addLayout(controls_layout)
        self.view = VerticalPitchTrackingView()
        main_layout.addWidget(self.view)

        self.setLayout(main_layout)

        self.audio_data = None
        self.sr = None
        self.timestamp = 0.0

        self.timer = QTimer()
        self.timer.start(30)
        self.timer.timeout.connect(self.on_timeout)

    def on_set_timestamp(self):
        self.timestamp = utils.get_duration_in_seconds(self.timestamp_input.text())
        self.update_scene()

    def on_refresh(self):
        threshold = self.threshold_input.value()
        window_width = self.window_width_input.value()
        self.view.vertical_pitch_tracking_scene.update_params(threshold, window_width)

    def on_timeout(self):
        if self.lock_to_timestamp_checkbox.isChecked():
            if self.audio_player.audio_state is not None and self.audio_player.current_timestamp != self.timestamp:
                self.timestamp = self.audio_player.current_timestamp
                self.timestamp_input.setText(utils.seconds_to_time_str(self.timestamp))
                self.update_scene()

    def update_scene(self):
        self.view.vertical_pitch_tracking_scene.set_timestamp(self.timestamp)
        self.view.vertical_pitch_tracking_scene.update()

    def set_audio_data(self, audio_data, sr):
        self.audio_data = audio_data
        self.sr = sr
        threshold = self.threshold_input.value()
        window_width = self.window_width_input.value()
        self.view.vertical_pitch_tracking_scene.set_data(self.audio_data, self.sr, threshold, window_width)

class VerticalPitchTrackingView(QGraphicsView):
    def __init__(self, *args, **kargs):
        super(VerticalPitchTrackingView, self).__init__(*args, **kargs)

        self.vertical_pitch_tracking_scene = VerticalPitchTrackingScene()
        self.setScene(self.vertical_pitch_tracking_scene)

        self.setMinimumHeight(150)

        self.setFrameShape(QFrame.Shape.NoFrame)

        self.horizontalScrollBar().setValue(1)
        self.setHorizontalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAlwaysOff)
        self.setVerticalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAlwaysOff)
        self.setViewportUpdateMode(QGraphicsView.ViewportUpdateMode.FullViewportUpdate)
        self.show()

class VerticalPitchTrackingScene(QGraphicsScene):
    def __init__(self, *args, **kargs):
        super(VerticalPitchTrackingScene, self).__init__(*args, **kargs)

        self.audio_data = None
        self.sr = None
        self.piptrack = None
        self.timestamp = 0.0

        self.setBackgroundBrush(Qt.GlobalColor.lightGray)

        self.rects = [None] * 88

    def set_data(self, audio_data, sr, threshold, window_width):
        self.pitches, self.magnitudes = librosa.piptrack(
            y=librosa.to_mono(y=audio_data),
            sr=sr,
            threshold=threshold,
            n_fft=window_width,
            win_length=window_width,
        )
        self.audio_data = audio_data
        self.sr = sr
        self.create_chart()

    def update_params(self, threshold, window_width):
        self.pitches, self.magnitudes = librosa.piptrack(
            y=librosa.to_mono(y=self.audio_data),
            sr=self.sr,
            threshold=threshold,
            n_fft=window_width,
            win_length=window_width,
        )
        self.update()

    def create_chart(self):
        self.clear()
        key_width = 20
        key_height = 100
        pen = QtGui.QPen(Qt.GlobalColor.black)
        brush = QtGui.QBrush(Qt.GlobalColor.white)
        for key in range(0, 88):
            self.rects[key] = self.addRect(key * key_width, 0, key_width, key_height, pen, brush)
            text = self.addText(utils.piano_key_to_note(key), QtGui.QFont("Courier New"))
            text.setDefaultTextColor(Qt.GlobalColor.black)
            text.setPos(key * key_width, key_height + 5)
        self.update()

    def set_timestamp(self, timestamp):
        self.timestamp = timestamp

    def update(self):
        duration = librosa.get_duration(y=self.audio_data, sr=self.sr)
        t = int(self.pitches.shape[1] * self.timestamp / duration)
        pitches = self.pitches[:,t]
        magnitudes = self.magnitudes[:,t]
        magnitudes_by_note = utils.group_by_note(pitches, magnitudes)
        max_magnitude = max(max(magnitudes), 50)
        #redunant, fix
        key_width = 20
        key_height = 100
        for key in range(0, 88):
            ms = magnitudes_by_note[key]
            h = key_height * (ms / max_magnitude)
            self.rects[key].setRect(key * key_width, key_height - h, key_width, h)

class MainView(QWidget):
    def __init__(self, audio_player, open_action, import_action, *args, **kargs):
        super(MainView, self).__init__(*args, **kargs)

        self.audio_view = None
        self.audio_player = audio_player
        layout = QBoxLayout(QBoxLayout.Direction.Down)

        self.open_button = QPushButton("Open...")
        self.open_button.clicked.connect(open_action.trigger)

        self.import_button = QPushButton("Import from YouTube...")
        self.import_button.clicked.connect(import_action.trigger)

        layout.addWidget(self.open_button)
        layout.addWidget(self.import_button)
        layout.addStretch(1)

        self.setLayout(layout)

    def show_audio(self, audio_data, on_loop_change):
        while self.layout().count() > 0:
            self.layout().takeAt(0)
        self.audio_view = audio_view.AudioWaveformView(audio_data, self.audio_player, on_loop_change, self)
        self.layout().addWidget(self.audio_view)

    def zoom_in(self):
        if self.audio_view != None:
            self.audio_view.zoom_in()

    def zoom_out(self):
        if self.audio_view != None:
            self.audio_view.zoom_out()

class EffectsDialog(QDialog):

    def __init__(self, play_rate, harmonic_only):
        super(EffectsDialog, self).__init__()

        layout = QGridLayout()

        self.rate_label = QLabel("Playback rate: ")
        self.rate_input = QDoubleSpinBox()
        self.rate_input.setValue(play_rate)
        self.rate_input.setMaximum(2.0)
        self.rate_input.setMinimum(0.1)

        layout.addWidget(self.rate_label, 0, 0)
        layout.addWidget(self.rate_input, 0, 1)

        self.harmonic_checkbox = QCheckBox("Enable HPSS")
        self.harmonic_checkbox.setChecked(harmonic_only)

        layout.addWidget(self.harmonic_checkbox, 1, 0, 1, 2)

        self.accept_button = QPushButton("OK")
        self.accept_button.clicked.connect(self.accept)
        self.cancel_button = QPushButton("Cancel")
        self.cancel_button.clicked.connect(self.reject)

        layout.addWidget(self.accept_button, 2, 0)
        layout.addWidget(self.cancel_button, 2, 1)

        self.setLayout(layout)

class MainWindow(QMainWindow):

    def __init__(self, audio_player: audio.AudioPlayer):
        super().__init__()

        self.audio_player = audio_player
        self.audio_data = None

        self.main_view = None
        self.key_pressed = dict()

        self.harmonic_only = False
        self.play_rate = 1.0

        self.resize(1024, 768)
        self.setWindowTitle("Noodler")

        self.playback_toolbar = QToolBar("Playback")
        self.back_action = self.playback_toolbar.addAction(icon("back.png"), "Back")
        self.back_action.triggered.connect(lambda: QApplication.postEvent(self, events.BackEvent()))
        self.play_action = self.playback_toolbar.addAction(icon("play.png"), "Play")
        self.play_action.triggered.connect(lambda: QApplication.postEvent(self, events.PlayEvent()))
        self.pause_action = self.playback_toolbar.addAction(icon("pause.png"), "Pause")
        self.pause_action.triggered.connect(lambda: QApplication.postEvent(self, events.PauseEvent()))
        self.addToolBar(Qt.ToolBarArea.TopToolBarArea, self.playback_toolbar)

        self.loop_toolbar = QToolBar("Selection")
        loop_layout = QGridLayout()
        self.range_start_widget = QLineEdit()
        self.range_start_widget.setMaximumWidth(50)
        loop_layout.addWidget(QLabel("Start:"), 0, 0)
        loop_layout.addWidget(self.range_start_widget, 0, 1)
        self.current_timestamp_widget = QLineEdit()
        self.current_timestamp_widget.setMaximumWidth(50)
        loop_layout.addWidget(QLabel("Current:"), 0, 2)
        loop_layout.addWidget(self.current_timestamp_widget, 0, 3)
        self.range_end_widget = QLineEdit()
        self.range_end_widget.setMaximumWidth(50)
        loop_layout.addWidget(QLabel("End:"), 1, 0)
        loop_layout.addWidget(self.range_end_widget, 1, 1)

        self.range_start_widget.returnPressed.connect(lambda: self.range_end_widget.setFocus())
        self.range_end_widget.returnPressed.connect(self.on_submit_range_selection)

        self.loop_checkbox = QCheckBox("Loop")
        self.loop_checkbox.setChecked(True)
        self.loop_checkbox.stateChanged.connect(self.on_set_loop_changed)
        loop_layout.addWidget(self.loop_checkbox, 0, 4)
        self.start_from_beginning_checkbox = QCheckBox("Start from Beginning")
        loop_layout.addWidget(self.start_from_beginning_checkbox, 1, 4)
        loop_layout.setColumnStretch(0, 0)
        loop_layout.setColumnStretch(1, 0)
        loop_layout.setColumnStretch(2, 0)
        loop_layout.setColumnStretch(3, 0)
        loop_layout.setColumnStretch(4, 1)
        loop_layout_widget = QWidget()
        loop_layout_widget.setLayout(loop_layout)
        loop_layout_widget.setSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Preferred)
        self.loop_toolbar.addWidget(loop_layout_widget)
        self.addToolBar(Qt.ToolBarArea.TopToolBarArea, self.loop_toolbar)

        self.move_toolbar = QToolBar("Move Selection")
        self.move_backward_by_duration_action = self.move_toolbar.addAction(icon("left_arrow.png"), "Move Selection Backward by Duration")
        self.move_backward_by_duration_action.triggered.connect(lambda: QApplication.postEvent(self, events.PauseEvent()))
        self.move_by_duration_amount = QDoubleSpinBox()
        self.move_by_duration_amount.setSuffix("s")
        self.move_toolbar.addWidget(self.move_by_duration_amount)
        self.move_forward_by_duration_action = self.move_toolbar.addAction(icon("right_arrow.png"), "Move Selection Forward by Duration")
        self.move_forward_by_duration_action.triggered.connect(lambda: QApplication.postEvent(self, events.PauseEvent()))

        self.move_backward_by_duration_action.triggered.connect(
            lambda: QApplication.postEvent(self, events.ShiftLoopEvent(-self.move_by_duration_amount.value())))
        self.move_forward_by_duration_action.triggered.connect(
            lambda: QApplication.postEvent(self, events.ShiftLoopEvent(self.move_by_duration_amount.value())))

        self.move_backward_by_percentage_action = self.move_toolbar.addAction(icon("left_arrow.png"), "Move Selection Backward by Percentage")
        self.move_backward_by_percentage_action.triggered.connect(lambda: QApplication.postEvent(self, events.PauseEvent()))
        self.move_by_percentage_amount = QDoubleSpinBox()
        self.move_by_percentage_amount.setSuffix("%")
        self.move_by_percentage_amount.setMinimum(0.0)
        self.move_by_percentage_amount.setMaximum(100.0)
        self.move_toolbar.addWidget(self.move_by_percentage_amount)
        self.move_forward_by_percentage_action = self.move_toolbar.addAction(icon("right_arrow.png"), "Move Selection Forward by Percentage")
        self.move_forward_by_percentage_action.triggered.connect(lambda: QApplication.postEvent(self, events.PauseEvent()))

        self.move_backward_by_percentage_action.triggered.connect(
            lambda: self.on_shift_by_percent(-self.move_by_percentage_amount.value()))
        self.move_forward_by_percentage_action.triggered.connect(
            lambda: self.on_shift_by_percent(self.move_by_percentage_amount.value()))

        self.addToolBar(Qt.ToolBarArea.TopToolBarArea, self.move_toolbar)

        self.vertical_pitch_tracking_dock_widget = QDockWidget("Vertical Pitch Tracking")
        self.addDockWidget(Qt.DockWidgetArea.BottomDockWidgetArea, self.vertical_pitch_tracking_dock_widget)
        self.vertical_pitch_tracking_widget = VerticalPitchTrackingWidget(self.audio_player)
        self.vertical_pitch_tracking_dock_widget.setWidget(self.vertical_pitch_tracking_widget)

        fileMenu = self.menuBar().addMenu("File")
        self.openAction = fileMenu.addAction("Open...", QtGui.QKeySequence.StandardKey.Open)
        self.openAction.triggered.connect(self.open)
        self.importAction = fileMenu.addAction("Import From YouTube...", QtGui.QKeySequence("Ctrl+Y"))

        self.importAction.triggered.connect(self.import_from_youtube)

        viewMenu = self.menuBar().addMenu("View")
        self.zoomInAction = viewMenu.addAction("Zoom In", QtGui.QKeySequence.StandardKey.ZoomIn)
        self.zoomOutAction = viewMenu.addAction("Zoom Out", QtGui.QKeySequence.StandardKey.ZoomOut)

        toolsMenu = self.menuBar().addMenu("Tools")
        setPlaybackRateAction = toolsMenu.addAction("Set Playback Rate", QtGui.QKeySequence("Ctrl+R"))
        setPlaybackRateAction.triggered.connect(self.set_playback_rate)
        effectsAction = toolsMenu.addAction("Effects...", QtGui.QKeySequence("Ctrl+E"))
        effectsAction.triggered.connect(self.set_effects)

        self.timer = QTimer()
        self.timer.start(15)
        self.timer.timeout.connect(self.handle_key_presses)

        self.main_view = MainView(self.audio_player, self.openAction, self.importAction, self)
        self.zoomInAction.triggered.connect(self.main_view.zoom_in)
        self.zoomOutAction.triggered.connect(self.main_view.zoom_out)
        self.setCentralWidget(self.main_view)

        self.setFocus()

    def is_key_pressed(self, key):
        return key in self.key_pressed and self.key_pressed[key]

    def handle_key_presses(self):
        if self.audio_player.ready and not self.audio_player.playing:

            if self.is_key_pressed(Qt.Key.Key_D):
                if self.is_key_pressed(Qt.Key.Key_Shift):
                    self.audio_player.set_current_timestamp(self.audio_player.current_timestamp + 0.1)
                else:
                    self.audio_player.set_current_timestamp(self.audio_player.current_timestamp + 0.01)

            if self.is_key_pressed(Qt.Key.Key_A):
                if self.is_key_pressed(Qt.Key.Key_Shift):
                    self.audio_player.set_current_timestamp(self.audio_player.current_timestamp - 0.1)
                else:
                    self.audio_player.set_current_timestamp(self.audio_player.current_timestamp - 0.01)

            if self.main_view != None:
                if self.is_key_pressed(Qt.Key.Key_E):
                    if self.is_key_pressed(Qt.Key.Key_Shift):
                        self.main_view.audio_view.audio_waveform_scene.shift_loop(0.1)
                    else:
                        self.main_view.audio_view.audio_waveform_scene.shift_loop(0.01)
                if self.is_key_pressed(Qt.Key.Key_Q):
                    if self.is_key_pressed(Qt.Key.Key_Shift):
                        self.main_view.audio_view.audio_waveform_scene.shift_loop(-0.1)
                    else:
                        self.main_view.audio_view.audio_waveform_scene.shift_loop(-0.01)

    def on_shift_by_percent(self, percent):
        if self.main_view.audio_view is not None:
            width = self.main_view.audio_view.audio_waveform_scene.loop_end - self.main_view.audio_view.audio_waveform_scene.loop_start
            amount = width * percent / 100.0
            QApplication.postEvent(self, events.ShiftLoopEvent(amount))

    def on_submit_range_selection(self):
        try:
            start_second = utils.get_duration_in_seconds(self.range_start_widget.text())
            end_second = utils.get_duration_in_seconds(self.range_end_widget.text())
            if end_second < start_second:
                end_second = start_second
            event = events.SetLoopEvent(start_second, end_second)
            QApplication.postEvent(self, event)
            self.setFocus()
        except ValueError:
            return

    def on_set_loop_changed(self, state):
        enabled = False
        if state == Qt.CheckState.Checked.value:
            enabled = True
        event = events.SetLoopConfiguration(enabled)
        QApplication.postEvent(self, event)

    def open(self):
        (path, result) = QFileDialog.getOpenFileName(None, "Open Audio File", "music", "Audio Files (*.mp4)");
        if not result:
            return
        self.load(path)

    def import_from_youtube(self):
        (url, result) = QInputDialog.getText(None, "Import From YouTube", "YouTube URL:")
        if not result:
            return
        if not os.path.exists("music"):
            os.makedirs("music")
        path = YouTube(url).streams.filter(file_extension='mp4').get_highest_resolution().download(output_path="music")
        self.load(path)

    def load(self, path):
        self.audio_data, sampling_rate = librosa.load(path, sr=None, mono=False)
        self.audio_player.set_audio_state(self.audio_data, sampling_rate, 1.0)
        self.main_view.show_audio(self.audio_data, self.on_loop_change)
        self.vertical_pitch_tracking_widget.set_audio_data(self.audio_data, sampling_rate)

    def on_loop_change(self, loop_start, loop_end):
        self.audio_player.set_start_timestamp(loop_start)
        self.audio_player.set_end_timestamp(loop_end)
        self.range_start_widget.setText(utils.seconds_to_time_str(loop_start))
        self.range_end_widget.setText(utils.seconds_to_time_str(loop_end))

    def set_playback_rate(self):
        (rate, _) = QInputDialog.getDouble(None, "Set Playback Rate", "Rate", value=1.0)
        # todo: handle the waiting period gracefully
        new_data = librosa.effects.time_stretch(y=self.audio_data, rate=rate)
        self.audio_player.set_audio_state(new_data, self.audio_player.audio_state.sampling_rate, rate)

    def set_effects(self):
        effects_dialog = EffectsDialog(self.play_rate, self.harmonic_only)
        result = effects_dialog.exec()
        if result == 1:
            self.play_rate = effects_dialog.rate_input.value()
            self.harmonic_only = effects_dialog.harmonic_checkbox.isChecked()
            new_data = self.audio_data
            if abs(self.play_rate- 1) > 0.01:
                new_data = librosa.effects.time_stretch(y=self.audio_data, sr=self.play_rate)
            if self.harmonic_only:
                new_data = librosa.effects.harmonic(y=new_data)
            self.audio_player.set_audio_state(new_data, self.audio_player.audio_state.sampling_rate, self.play_rate)
            self.vertical_pitch_tracking_widget.set_audio_data(self.audio_data, self.audio_player.audio_state.sampling_rate)

    def keyReleaseEvent(self, a0: QtGui.QKeyEvent) -> None:
        self.key_pressed[a0.key()] = False
        return super().keyReleaseEvent(a0)

    def keyPressEvent(self, a0: QtGui.QKeyEvent) -> None:
        self.key_pressed[a0.key()] = True
        if not self.audio_player.ready:
            return super().keyPressEvent(a0)
        if a0.key() == Qt.Key.Key_Space:
            if self.audio_player.playing:
                QApplication.postEvent(self, events.PauseEvent())
            else:
                QApplication.postEvent(self, events.PlayEvent())
        return None

    def customEvent(self, event: QEvent):
        if event.type() == events.SetLoopEvent.TYPE:
            if not self.audio_player.playing and self.main_view.audio_view is not None:
                self.main_view.audio_view.audio_waveform_scene.set_loop(event.get_start(), event.get_end())
            return

        elif event.type() == events.ShiftLoopEvent.TYPE:
            if not self.audio_player.playing and self.main_view.audio_view is not None:
                self.main_view.audio_view.audio_waveform_scene.shift_loop(event.get_amount())
            return
        elif event.type() == events.PlayEvent.TYPE:
            if self.audio_player.audio_state is not None and not self.audio_player.playing:
                if self.start_from_beginning_checkbox.isChecked():
                    self.audio_player.set_current_timestamp(self.audio_player.start_timestamp)
                self.audio_player.play()
            return
        elif event.type() == events.PauseEvent.TYPE:
            self.audio_player.stop()
            return
        elif event.type() == events.BackEvent.TYPE:
            if self.main_view.audio_view is not None:
                if self.audio_player.playing:
                    self.audio_player.stop()
                    self.audio_player.set_current_timestamp(self.audio_player.start_timestamp)
                    self.audio_player.play()
                else:
                    self.audio_player.set_current_timestamp(self.main_view.audio_view.audio_waveform_scene.loop_start)
        elif event.type() == events.SetLoopConfiguration.TYPE:
            self.audio_player.set_loop(event.get_loop_enabled())
        return super().customEvent(event)


#!/usr/bin/env python3
# Copyright (C) Alibaba Group Holding Limited.

""" Transformations. """

import torch
import math

import torchvision.transforms._functional_video as F
from torchvision.transforms import Lambda, Compose
import random
import numbers

class ColorJitter(object):
    """
    Modified from https://github.com/TengdaHan/DPC/blob/master/utils/augmentation.py.
    Randomly change the brightness, contrast and saturation of an image.
    Args:
        brightness (float or tuple of float (min, max)): How much to jitter brightness.
            brightness_factor is chosen uniformly from [max(0, 1 - brightness), 1 + brightness]
            or the given [min, max]. Should be non negative numbers.
        contrast (float or tuple of float (min, max)): How much to jitter contrast.
            contrast_factor is chosen uniformly from [max(0, 1 - contrast), 1 + contrast]
            or the given [min, max]. Should be non negative numbers.
        saturation (float or tuple of float (min, max)): How much to jitter saturation.
            saturation_factor is chosen uniformly from [max(0, 1 - saturation), 1 + saturation]
            or the given [min, max]. Should be non negative numbers.
        hue (float or tuple of float (min, max)): How much to jitter hue.
            hue_factor is chosen uniformly from [-hue, hue] or the given [min, max].
            Should have 0<= hue <= 0.5 or -0.5 <= min <= max <= 0.5.
        grayscale (float): possibility to transform the video to grayscale.
            Should have a value range of [0, 1]
        consistent  (bool): indicates whether or not to keep all the color transformations consistent for all the frames.
        shuffle     (bool): indicates whether or not to shuffle the sequence of the augmentations.
        gray_first  (bool): indicates whether or not to put grayscale transform first.
    """
    def __init__(
        self, brightness=0, contrast=0, saturation=0, hue=0, grayscale=0, color=0., consistent=False, shuffle=True, gray_first=True
    ):
        self.brightness = self._check_input(brightness, 'brightness')
        self.contrast = self._check_input(contrast, 'contrast')
        self.saturation = self._check_input(saturation, 'saturation')
        self.hue = self._check_input(hue, 'hue', center=0, bound=(-0.5, 0.5),
                                     clip_first_on_zero=False)

        self.color = color
        self.grayscale = grayscale
        self.consistent = consistent
        self.shuffle = shuffle
        self.gray_first = gray_first

    def _check_input(self, value, name, center=1, bound=(0, float('inf')), clip_first_on_zero=True):
        if isinstance(value, numbers.Number):
            if value < 0:
                raise ValueError("If {} is a single number, it must be non negative.".format(name))
            value = [center - float(value), center + float(value)]
            if clip_first_on_zero:
                value[0] = max(value[0], 0.0)
        elif isinstance(value, (tuple, list)) and len(value) == 2:
            if not bound[0] <= value[0] <= value[1] <= bound[1]:
                raise ValueError("{} values should be between {}".format(name, bound))
        else:
            raise TypeError("{} should be a single number or a list/tuple with lenght 2.".format(name))

        # if value is 0 or (1., 1.) for brightness/contrast/saturation
        # or (0., 0.) for hue, do nothing
        if value[0] == value[1] == center:
            value = None
        return value

    def _get_transform(self, T, device):
        """Get a randomized transform to be applied on image.
        Arguments are same as that of __init__.
        Arg:
            T (int): number of frames. Used when consistent = False.
        Returns:
            Transform which randomly adjusts brightness, contrast and
            saturation in a random order.
        """
        transforms = []
        if random.uniform(0, 1) < self.color:
            if self.brightness is not None:
                if self.consistent:
                    brightness_factor = random.uniform(self.brightness[0], self.brightness[1])
                else:
                    brightness_factor = torch.empty([1, T, 1, 1], device=device).uniform_(self.brightness[0], self.brightness[1])
                transforms.append(Lambda(lambda frame: adjust_brightness(frame, brightness_factor)))

            if self.contrast is not None:
                if self.consistent:
                    contrast_factor = random.uniform(self.contrast[0], self.contrast[1])
                else:
                    contrast_factor = torch.empty([1, T, 1, 1], device=device).uniform_(self.contrast[0], self.contrast[1])
                transforms.append(Lambda(lambda frame: adjust_contrast(frame, contrast_factor)))

            if self.saturation is not None:
                if self.consistent:
                    saturation_factor = random.uniform(self.saturation[0], self.saturation[1])
                else:
                    saturation_factor = torch.empty([1, T, 1, 1], device=device).uniform_(self.saturation[0], self.saturation[1])
                transforms.append(Lambda(lambda frame: adjust_saturation(frame, saturation_factor)))

            if self.hue is not None:
                if self.consistent:
                    hue_factor = random.uniform(self.hue[0], self.hue[1])
                else:
                    hue_factor = torch.empty([T, 1, 1], device=device).uniform_(self.hue[0], self.hue[1])
                transforms.append(Lambda(lambda frame: adjust_hue(frame, hue_factor)))

            if self.shuffle:
                random.shuffle(transforms)

        if random.uniform(0, 1) < self.grayscale:
            gray_transform = Lambda(lambda frame: rgb_to_grayscale(frame))
            if self.gray_first:
                transforms.insert(0, gray_transform)
            else:
                transforms.append(gray_transform)

        transform = Compose(transforms)

        return transform

    def __call__(self, clip):
        """
        Args:
            clip (torch.tensor): Size is (C, T, H, W)
        Return:
            clip (torch.tensor): Size is (C, T, H, W)
        """

        raw_shape = clip.shape #(C, T, H, W)
        device = clip.device
        T = raw_shape[1]
        transform = self._get_transform(T, device)
        clip = transform(clip)
        assert clip.shape == raw_shape
        return clip #(C, T, H, W)

    def __repr__(self):
        format_string = self.__class__.__name__ + '('
        format_string += 'brightness={0}'.format(self.brightness)
        format_string += ', contrast={0}'.format(self.contrast)
        format_string += ', saturation={0}'.format(self.saturation)
        format_string += ', hue={0})'.format(self.hue)
        format_string += ', grayscale={0})'.format(self.grayscale)
        return format_string

def _is_tensor_a_torch_image(input):
    return input.ndim >= 2

def _blend(img1, img2, ratio):
    # type: (Tensor, Tensor, float) -> Tensor
    bound = 1 if img1.dtype in [torch.half, torch.float32, torch.float64] else 255
    return (ratio * img1 + (1 - ratio) * img2).clamp(0, bound).to(img1.dtype)

def rgb_to_grayscale(img):
    # type: (Tensor) -> Tensor
    """Convert the given RGB Image Tensor to Grayscale.
    For RGB to Grayscale conversion, ITU-R 601-2 luma transform is performed which
    is L = R * 0.2989 + G * 0.5870 + B * 0.1140
    Args:
        img (Tensor): Image to be converted to Grayscale in the form [C, H, W].
    Returns:
        Tensor: Grayscale image.
        Args:
            clip (torch.tensor): Size is (T, H, W, C)
        Return:
            clip (torch.tensor): Size is (T, H, W, C)
    """
    orig_dtype = img.dtype
    rgb_convert = torch.tensor([0.299, 0.587, 0.114])

    assert img.shape[0] == 3, "First dimension need to be 3 Channels"
    if img.is_cuda:
        rgb_convert = rgb_convert.to(img.device)

    img = img.float().permute(1,2,3,0).matmul(rgb_convert).to(orig_dtype)
    return torch.stack([img, img, img], 0)

def _rgb2hsv(img):
    r, g, b = img.unbind(0)

    maxc, _ = torch.max(img, dim=0)
    minc, _ = torch.min(img, dim=0)

    eqc = maxc == minc
    cr = maxc - minc
    s = cr / torch.where(eqc, maxc.new_ones(()), maxc)
    cr_divisor = torch.where(eqc, maxc.new_ones(()), cr)
    rc = (maxc - r) / cr_divisor
    gc = (maxc - g) / cr_divisor
    bc = (maxc - b) / cr_divisor

    hr = (maxc == r) * (bc - gc)
    hg = ((maxc == g) & (maxc != r)) * (2.0 + rc - bc)
    hb = ((maxc != g) & (maxc != r)) * (4.0 + gc - rc)
    h = (hr + hg + hb)
    h = torch.fmod((h / 6.0 + 1.0), 1.0)
    return torch.stack((h, s, maxc))

def _hsv2rgb(img):
    l = len(img.shape)
    h, s, v = img.unbind(0)
    i = torch.floor(h * 6.0)
    f = (h * 6.0) - i
    i = i.to(dtype=torch.int32)

    p = torch.clamp((v * (1.0 - s)), 0.0, 1.0)
    q = torch.clamp((v * (1.0 - s * f)), 0.0, 1.0)
    t = torch.clamp((v * (1.0 - s * (1.0 - f))), 0.0, 1.0)
    i = i % 6

    if l == 3:
        tmp = torch.arange(6)[:, None, None]
    elif l == 4:
        tmp = torch.arange(6)[:, None, None, None]

    if img.is_cuda:
        tmp = tmp.to(img.device)

    mask = i == tmp #(H, W) == (6, H, W)

    a1 = torch.stack((v, q, p, p, t, v))
    a2 = torch.stack((t, v, v, q, p, p))
    a3 = torch.stack((p, p, t, v, v, q))
    a4 = torch.stack((a1, a2, a3)) #(3, 6, H, W)

    if l == 3:
        return torch.einsum("ijk, xijk -> xjk", mask.to(dtype=img.dtype), a4) #(C, H, W)
    elif l == 4:
        return torch.einsum("itjk, xitjk -> xtjk", mask.to(dtype=img.dtype), a4) #(C, T, H, W)

def adjust_brightness(img, brightness_factor):
    # type: (Tensor, float) -> Tensor
    if not _is_tensor_a_torch_image(img):
        raise TypeError('tensor is not a torch image.')

    return _blend(img, torch.zeros_like(img), brightness_factor)

def adjust_contrast(img, contrast_factor):
    # type: (Tensor, float) -> Tensor
    if not _is_tensor_a_torch_image(img):
        raise TypeError('tensor is not a torch image.')

    mean = torch.mean(rgb_to_grayscale(img).to(torch.float), dim=(-4, -2, -1), keepdim=True)

    return _blend(img, mean, contrast_factor)

def adjust_saturation(img, saturation_factor):
    # type: (Tensor, float) -> Tensor
    if not _is_tensor_a_torch_image(img):
        raise TypeError('tensor is not a torch image.')

    return _blend(img, rgb_to_grayscale(img), saturation_factor)

def adjust_hue(img, hue_factor):
    """Adjust hue of an image.
    The image hue is adjusted by converting the image to HSV and
    cyclically shifting the intensities in the hue channel (H).
    The image is then converted back to original image mode.
    `hue_factor` is the amount of shift in H channel and must be in the
    interval `[-0.5, 0.5]`.
    See `Hue`_ for more details.
    .. _Hue: https://en.wikipedia.org/wiki/Hue
    Args:
        img (Tensor): Image to be adjusted. Image type is either uint8 or float.
        hue_factor (float):  How much to shift the hue channel. Should be in
            [-0.5, 0.5]. 0.5 and -0.5 give complete reversal of hue channel in
            HSV space in positive and negative direction respectively.
            0 means no shift. Therefore, both -0.5 and 0.5 will give an image
            with complementary colors while 0 gives the original image.
    Returns:
         Tensor: Hue adjusted image.
    """
    if isinstance(hue_factor, float) and  not(-0.5 <= hue_factor <= 0.5):
        raise ValueError('hue_factor ({}) is not in [-0.5, 0.5].'.format(hue_factor))
    elif isinstance(hue_factor, torch.Tensor) and not ((-0.5 <= hue_factor).sum() == hue_factor.shape[0] and (hue_factor <= 0.5).sum() == hue_factor.shape[0]):
        raise ValueError('hue_factor ({}) is not in [-0.5, 0.5].'.format(hue_factor))

    if not _is_tensor_a_torch_image(img):
        raise TypeError('tensor is not a torch image.')

    orig_dtype = img.dtype
    if img.dtype == torch.uint8:
        img = img.to(dtype=torch.float32) / 255.0

    img = _rgb2hsv(img)
    h, s, v = img.unbind(0)
    h += hue_factor
    h = h % 1.0
    img = torch.stack((h, s, v))
    img_hue_adj = _hsv2rgb(img)

    if orig_dtype == torch.uint8:
        img_hue_adj = (img_hue_adj * 255.0).to(dtype=orig_dtype)

    return img_hue_adj

class AutoResizedCropVideo(object):
    def __init__(
            self,
            size,
            scale=(0.08, 1.0),
            interpolation_mode="bilinear",
            mode = "cc"
    ):
        # mode how many clips return
        if isinstance(size, tuple):
            assert len(size) == 2, "size should be tuple (height, width)"
            self.size = size
        else:
            self.size = (size, size)

        self.interpolation_mode = interpolation_mode
        self.scale = scale
        self.mode = mode
        self.idx = 0

    def set_spatial_index(self, idx):
        self.idx = idx

    def get_crop(self, clip):
        crop_mode = self.mode[self.idx:self.idx+2]

        scale = random.uniform(*self.scale)

        # Get the crop size for the scale cropping
        _, _, image_height, image_width = clip.shape

        min_length = min(image_width, image_height)
        crop_size = int(min_length * scale)

        center_x = image_width // 2
        center_y = image_height // 2
        box_half = crop_size // 2
        th = crop_size
        tw = crop_size

        if crop_mode == "cc":
            x1 = center_x - box_half
            y1 = center_y - box_half
            x2 = center_x + box_half
            y2 = center_y + box_half
        elif crop_mode == "ll":
            x1 = 0
            y1 = center_y - box_half
            x2 = crop_size
            y2 = center_y + box_half
        elif crop_mode == "rr":
            x1 = image_width - crop_size
            y1 = center_y - box_half
            x2 = image_width
            y2 = center_y + box_half
        elif crop_mode == "tl":
            x1 = 0
            y1 = 0
            x2 = crop_size
            y2 = crop_size
        elif crop_mode == "tr":
            x1 = image_width - crop_size
            y1 = 0
            x2 = image_width
            y2 = crop_size
        elif crop_mode == "bl":
            x1 = 0
            y1 = image_height - crop_size
            x2 = crop_size
            y2 = image_height
        elif crop_mode == "br":
            x1 = image_width - crop_size
            y1 = image_height - crop_size
            x2 = image_width
            y2 = image_height

        crop = F.resized_crop(clip, y1, x1, th, tw, self.size, self.interpolation_mode)
        return crop

    def __call__(self, clip):
        """
        Args:
            clip (torch.tensor): Video clip to be cropped. Size is (C, T, H, W)
        Returns:
            torch.tensor: randomly cropped/resized video clip.
                size is (C, T, H, W)
        """
        if self.idx == -1:
            # return self.get_random_crop(clip)
            pass
        else:
            return self.get_crop(clip)

class KineticsResizedCrop(object):
    def __init__(
        self,
        short_side_range,
        crop_size,
        num_spatial_crops=1,
    ):
        self.idx = -1
        self.short_side_range = short_side_range
        self.crop_size = int(crop_size)
        self.num_spatial_crops = num_spatial_crops

    def _get_controlled_crop(self, clip):
        _, _, clip_height, clip_width = clip.shape

        length = self.short_side_range[0]

        if clip_height < clip_width:
            new_clip_height = int(length)
            new_clip_width = int(clip_width / clip_height * new_clip_height)
            new_clip = torch.nn.functional.interpolate(
                clip, size=(new_clip_height, new_clip_width), mode="bilinear"
            )
        else:
            new_clip_width = int(length)
            new_clip_height = int(clip_height / clip_width * new_clip_width)
            new_clip = torch.nn.functional.interpolate(
                clip, size=(new_clip_height, new_clip_width), mode="bilinear"
            )
        x_max = int(new_clip_width - self.crop_size)
        y_max = int(new_clip_height - self.crop_size)
        if self.num_spatial_crops == 1:
            x = x_max // 2
            y = y_max // 2
        elif self.num_spatial_crops == 3:
            if self.idx == 0:
                if new_clip_width == length:
                    x = x_max // 2
                    y = 0
                elif new_clip_height == length:
                    x = 0
                    y = y_max // 2
            elif self.idx == 1:
                x = x_max // 2
                y = y_max // 2
            elif self.idx == 2:
                if new_clip_width == length:
                    x = x_max // 2
                    y = y_max
                elif new_clip_height == length:
                    x = x_max
                    y = y_max // 2
        return new_clip[:, :, y:y+self.crop_size, x:x+self.crop_size]

    def _get_random_crop(self, clip):
        _, _, clip_height, clip_width = clip.shape

        if clip_height < clip_width:
            new_clip_height = int(random.uniform(*self.short_side_range))
            new_clip_width = int(clip_width / clip_height * new_clip_height)
            new_clip = torch.nn.functional.interpolate(
                clip, size=(new_clip_height, new_clip_width), mode="bilinear"
            )
        else:
            new_clip_width = int(random.uniform(*self.short_side_range))
            new_clip_height = int(clip_height / clip_width * new_clip_width)
            new_clip = torch.nn.functional.interpolate(
                clip, size=(new_clip_height, new_clip_width), mode="bilinear"
            )
        x_max = int(new_clip_width - self.crop_size)
        y_max = int(new_clip_height - self.crop_size)
        x = int(random.uniform(0, x_max))
        y = int(random.uniform(0, y_max))
        return new_clip[:, :, y:y+self.crop_size, x:x+self.crop_size]

    def set_spatial_index(self, idx):
        self.idx = idx

    def __call__(self, clip):
        if self.idx == -1:
            return self._get_random_crop(clip)
        else:
            return self._get_controlled_crop(clip)


import asyncio, discord
from discord.ext import commands
import time

client = commands.Bot(command_prefix='-')
status = False
channel = ''

@client.command()
async def get_channel(ctx):
    global channel
    for channel in client.get_all_channels():
        if 'bump' in channel.name:
            channel = client.get_channel(channel.id)

@client.event
async def bump():
    print(status, channel)
    while status == True:
        await channel.send("!d bump")
        await asyncio.sleep(7200)

@client.event
async def on_ready():
    print('bot online')
    await get_channel('x')
    await bump()

@client.event
async def on_guild_join(guild):
    await helpsheet('x')
    await get_channel('x')
    await bump()

@client.command()
async def bumpbot(ctx):
    global status
    on_off = True if str(ctx.message.content.split(' ')[1]) == 'on' else False
    msg_color = discord.Color.green() if on_off == True else discord.Color.red()
    status = on_off # change bot on/off status

    embed = discord.Embed(title=("BumpBot is now "+str(status)), color = msg_color)
    await channel.send(embed=embed) # send embedded msg to channel
    await bump()

@client.command()
async def helpsheet(ctx):
    embed = discord.Embed(title="__BumpBot Help Sheet__", description="Please refer to this sheet when using the BumpBot :)", color=discord.Color.purple()) #,color=Hex code
    embed.add_field(name="`-helpsheet`", value="Brings up this sheet!")
    embed.add_field(name="`-bumpbot [on/off]`", value="Turn the bot on or off.")
    await channel.send(embed=embed)

client.run('<key>')


from telebot import TeleBot, types

from config.config import ConfigYamlParser
from state.bot_state import BotState

# для простоты захардкодим путь до конфиг файла
_CONFIG_FILE_NAME = "./config/bot.yaml"
_config = ConfigYamlParser(config_file_name=_CONFIG_FILE_NAME)
_bot_state = BotState()
_bot = TeleBot(_config.get_bot_token())

def _show_start_message(chat_id):
    _bot.send_message(chat_id=chat_id,
                      text="Привет!\nЯ тестовый чат-бот для заказа еды.")

def _show_products_list_message(chat_id):
    markup = types.ReplyKeyboardMarkup(resize_keyboard=True)
    for pr_name in _config.get_chat_products():
        markup.add(types.KeyboardButton(pr_name))
    _bot.send_message(chat_id=chat_id,
                      text="У вас есть возможность заказать несколько блюд.",
                      reply_markup=markup)

def _product_selected_reply(message):
    _bot_state.trigger("product_selected", message.text)

    markup = types.ReplyKeyboardMarkup(resize_keyboard=True)
    for pr_size in _config.get_chat_products_sizes():
        markup.add(types.KeyboardButton(pr_size))
    _bot.send_message(chat_id=message.chat.id,
                      text=f"Вы выбрали продукт '{_bot_state.get_product_name().lower()}'.\nКакой размер вы хотите?",
                      reply_markup=markup)

def _product_size_selected_reply(message):
    _bot_state.trigger("size_selected", message.text)

    markup = types.ReplyKeyboardMarkup(resize_keyboard=True)
    for payment_method in _config.get_chat_payment_method():
        markup.add(types.KeyboardButton(payment_method))
    _bot.send_message(chat_id=message.chat.id,
                      text=f"Вы выбрали продукт '{_bot_state.get_product_name().lower()}'"
                           f" размера '{_bot_state.get_product_size().lower()}'.\nКакой метод оплаты?",
                      reply_markup=markup)

def _payment_selected_reply(message):
    _bot_state.trigger("payment_selected", message.text)

    _bot.send_message(chat_id=message.chat.id,
                      text=f"Вы выбрали продукт '{_bot_state.get_product_name().lower()}'"
                           f" размера '{_bot_state.get_product_size().lower()}."
                           f" Оплата {_bot_state.get_payment_method().lower()}.\nСпасибо за заказ")

    _show_products_list_message(message.chat.id)

@_bot.message_handler(commands=['start'])
def _start_message(message):
    chat_id = message.chat.id
    _show_start_message(chat_id)
    _show_products_list_message(chat_id)

@_bot.message_handler(content_types="text")
def _message_reply(message):
    if _bot_state.is_waiting_for_product_name():
        _product_selected_reply(message)
    elif _bot_state.is_waiting_for_product_size():
        _product_size_selected_reply(message)
    elif _bot_state.is_waiting_for_payment_method():
        _payment_selected_reply(message)

def telegram_bot_start():
    _bot.infinity_polling()


# -*- coding: UTF-8 -*-
import numpy as np
import pandas as pd
import sklearn.metrics as metrics
from sklearn import linear_model
from sklearn.cross_validation import train_test_split
from sklearn import ensemble
from sklearn.grid_search import GridSearchCV
import threading
import logging

from services import wcar_service
from services import car_type_service
from utils import math_utils
from utils import time_utils
import config
from services.regression_service import RegressionService

"""car price regression of car series

Attribute:
----------
hedge_rates: car series hedge rates list, used to filter unreasonable data.
"""
class CarSeriesRegressionService(RegressionService):
	hedge_rates = None

	@classmethod
	def _init_series_estimator(cls, series_id, province_id):
		# start to init series estimator
		logging.info("init series estimator start")
		# prepare car series hedge rates
		cls.hedge_rates = car_type_service.get_hedge_rates_by_series_id(series_id)

		# prepare train data set
		logging.debug("before prepare data: %f" % time_utils.milli_time())
		cls.data_frame, i = cls._prepare_data(series_id, province_id)
		logging.debug("after prepare data: %f" % time_utils.milli_time())
		if len(cls.data_frame) < config.MIN_TRAIN_NUM:
			cls.est = False
			return

		# split data into train part and test part.
		logging.debug("before train and test data splitting: %f" % time_utils.milli_time())
		X_train, X_test, y_train, y_test = train_test_split(cls.data_frame.drop(['deal_price'], axis=1), cls.data_frame.deal_price, test_size=0.1, random_state=20)
		logging.debug("after train and test data splitting: %f" % time_utils.milli_time())

		# initialize regression method.
		tuned_parameters = {}
		regressor = linear_model.BayesianRidge()
		clf = GridSearchCV(regressor, cv=4, param_grid=tuned_parameters, scoring=cls.loss)
		logging.debug("before fit: %f" % time_utils.milli_time())
		preds = clf.fit(X_train, y_train)
		logging.debug("after fit: %f" % time_utils.milli_time())
		cls.est = clf.best_estimator_

		# calculate test score
		cls.test_score = math_utils.mean_loss_percentage(cls.est.predict(X_test), y_test)
		if cls.test_score > config.MAX_ALLOWED_PERCENTAGE_ERROR: cls.test_score = config.MAX_ALLOWED_PERCENTAGE_ERROR

		logging.info("init series estimator end")

	"""estimate car price"""
	@classmethod
	def estimate(cls, car_deal):
		result = {}

		# init estimator, if failed, return empty result
		cls._init_series_estimator(car_deal.series_id, car_deal.province_id)
		if not cls.est: return result

		# find predict X array
		predict_X = cls.prepare_predict_X(car_deal)
		predict_y = cls.est.predict(predict_X)
		result['price'] = int(predict_y)
		result['min_price'] = int(result['price'] * (1 - cls.test_score))
		result['max_price'] = int(result['price'] * (1 + cls.test_score))
		result['price_c'] = result['price']
		result['min_price_c'] = result['min_price']
		result['max_price_c'] = result['max_price']
		result['predict_method'] = 'linear regression'
		result['test_score'] = cls.test_score
		return result

	"""prepare training data_dict"""
	@classmethod
	def _prepare_data(cls, series_id, province_id):
		i = 1
		deal_province_offset = 0
		# try to filter training data by series_id, sale_name and province_id,
		# if training set is not enough, remove province_id constraint
		while i >= 0:
			if (i & (1 << deal_province_offset)) > 0:
				tmp_province_id = province_id
			else:
				tmp_province_id = None
			data_frame = wcar_service.get_car_deal_data_by_series_id_and_province_id(series_id, tmp_province_id)
			# filter data
			logging.debug("before filtering data: %f" % time_utils.milli_time())
			data_frame = cls.filter_data(data_frame, cls.hedge_rates)
			logging.debug("after filtering data: %f" % time_utils.milli_time())
			if len(data_frame) >= config.MIN_TRAIN_NUM:
				break
			i = i - 1
		# time consuming here
		# logging.debug("before filled blank data: %f" % time_utils.milli_time())
		# data_frame = cls.filled_blank_data(data_frame)
		# logging.debug("after filled blank data: %f" % time_utils.milli_time())
		return data_frame, i


import os
import base64

# import mock while in development/debug mode
if ("DEBUGMODE" in os.environ):
  import sys
  import fake_rpi
  sys.modules['picamera'] = fake_rpi.picamera
else:
  from picamera import PiCamera

class CameraModule:

  """
  CameraModule class interfaces camera component and includes:
  - functionality to capture with specifications
  - get base64 format of the image
  """

  def __init__(self, resolution="MAX", rotate=0, capturefile="capture.jpg"):
    self.camera = PiCamera()

    # check for resolution
    if (resolution == "MAX"):
      self.camera.resolution = self.camera.MAX_RESOLUTION
    elif (type(resolution) is tuple):
      self.camera.resolution = resolution
    else:
      raise ValueError("invalid resolution value supplied. expecting MAX or tuple of (width, height) value in pixels")

    # check for rotate value
    if (type(rotate) is not int):
      raise ValueError("value of rotate must be int")
    elif(rotate < 0 or rotate > 360):
      raise ValueError("value of rotate must be between 0 and 360")
    else:
      self.camera.rotation = rotate

    # check for capturefile
    if((type(capturefile) is not str)):
      raise ValueError("check the value of capturefile")
    elif(not capturefile.endswith(".jpg") and not capturefile.endswith(".png")):
      raise ValueError("capturefile extension must be either .png or .jpg")
    else:
      self.capturefile = capturefile

  # method to capture from camera
  def capture(self, capturefile=None):
    if(capturefile is None):
      capturefile = self.capturefile

    self.camera.capture(capturefile)

  # method to get base64 form of the captured image
  def get_base64_image(self):
    return str(base64.b64encode(open(self.capturefile,"rb").read()).decode("ascii"))

  # method to close and cleanup camera resource
  def cleanup(self):
    self.camera.close()


# -*- coding: utf-8 -*-
"""
Задание 19.3a

Создать функцию send_command_to_devices, которая отправляет
список указанных команды show на разные устройства в параллельных потоках,
а затем записывает вывод команд в файл. Вывод с устройств в файле может быть в любом порядке.

Параметры функции:
* devices - список словарей с параметрами подключения к устройствам
* commands_dict - словарь в котором указано на какое устройство отправлять какие команды. Пример словаря - commands
* filename - имя файла, в который будут записаны выводы всех команд
* limit - максимальное количество параллельных потоков (по умолчанию 3)

Функция ничего не возвращает.

Вывод команд должен быть записан в файл в таком формате (перед выводом каждой команды надо написать имя хоста и саму команду):

R2#sh arp
Protocol  Address          Age (min)  Hardware Addr   Type   Interface
Internet  192.168.100.1          87   aabb.cc00.6500  ARPA   Ethernet0/0
Internet  192.168.100.2           -   aabb.cc00.6600  ARPA   Ethernet0/0
R1#sh ip int br
Interface                  IP-Address      OK? Method Status                Protocol
Ethernet0/0                192.168.100.1   YES NVRAM  up                    up
Ethernet0/1                192.168.200.1   YES NVRAM  up                    up
R1#sh arp
Protocol  Address          Age (min)  Hardware Addr   Type   Interface
Internet  10.30.0.1               -   aabb.cc00.6530  ARPA   Ethernet0/3.300
Internet  10.100.0.1              -   aabb.cc00.6530  ARPA   Ethernet0/3.100
R3#sh ip int br
Interface                  IP-Address      OK? Method Status                Protocol
Ethernet0/0                192.168.100.3   YES NVRAM  up                    up
Ethernet0/1                unassigned      YES NVRAM  administratively down down
R3#sh ip route | ex -

Gateway of last resort is not set

      10.0.0.0/8 is variably subnetted, 4 subnets, 2 masks
O        10.1.1.1/32 [110/11] via 192.168.100.1, 07:12:03, Ethernet0/0
O        10.30.0.0/24 [110/20] via 192.168.100.1, 07:12:03, Ethernet0/0

Порядок команд в файле может быть любым.

Для выполнения задания можно создавать любые дополнительные функции, а также использовать функции созданные в предыдущих заданиях.

Проверить работу функции на устройствах из файла devices.yaml и словаре commands
"""

# Этот словарь нужен только для проверки работа кода, в нем можно менять IP-адреса
# тест берет адреса из файла devices.yaml

commands = {
    "192.168.100.3": ["sh ip int br", "sh ip route | ex -"],
    "ios-xe-mgmt.cisco.com": ["sh int desc", "sh ip int br"],
    "192.168.100.2": ["sh int desc"]
}

from datetime import datetime
import time
from itertools import repeat
from concurrent.futures import ThreadPoolExecutor
import logging
from pprint import pprint

import netmiko
import yaml

logging.getLogger('paramiko').setLevel(logging.WARNING)

logging.basicConfig(
    format = '%(threadName)s %(name)s %(levelname)s: %(message)s',
    level=logging.INFO)

def send_show_command_to_devices(device, command):
	final = []
	start_msg = '===> {} Connection: {}'
	received_msg = '<=== {} Received:   {}'
	ip = device['host']
	if commands.get(device['host']):
		command = commands[device['host']]
	logging.info(start_msg.format(datetime.now().time(), ip))

	with netmiko.ConnectHandler(**device) as ssh:
		ssh.enable()
		output1 = ssh.send_command(strip_command=False, command_string=command[0])
		final.append(device['host']+'#'+output1+'\n')
		output2 = ssh.send_command(strip_command=False, command_string=command[1])
		final.append(device['host']+'#'+output2+'\n')
		logging.info(received_msg.format(datetime.now().time(), ip))
	return final

with open('devices2.yaml') as f:
	devices = yaml.safe_load(f)

with ThreadPoolExecutor(max_workers=3) as executor:
	result = executor.map(send_show_command_to_devices, devices, commands)
	for output in result:
		pprint(output)
		f = open('show.txt', 'a')
		for line in output:
			f.write(line)
'''
ios-xe-mgmt.cisco.com#sh int desc
Interface                      Status         Protocol Description
Gi1                            up             up       MANAGEMENT INTERFACE - DON'T TOUCH ME
Gi2                            admin down     down     Network Interface
Gi3                            admin down     down     Network Interface
Lo15                           up             up       Added By KaussezPausies
Lo100                          up             up       Added By KaussezPausies
ios-xe-mgmt.cisco.com#sh ip int br
Interface              IP-Address      OK? Method Status                Protocol
GigabitEthernet1       10.10.20.48     YES NVRAM  up                    up
GigabitEthernet2       unassigned      YES NVRAM  administratively down down
GigabitEthernet3       unassigned      YES NVRAM  administratively down down
Loopback15             unassigned      YES unset  up                    up
Loopback100            172.16.55.1     YES other  up                    up
'''


import platform
import argparse
from packaging import version

parser = argparse.ArgumentParser()
parser.add_argument('--min-version', help='Minimum Python version')
args = parser.parse_args()

actual_py=version.parse(platform.python_version())
min_py=version.parse(args.min_version)

if (min_py > actual_py):
    print(f"Error: Python version is {actual_py}, but required {min_py} or higher!")
    exit(1)


import json
import os
import traceback
from django.conf import settings

import requests
from bs4 import BeautifulSoup as bs

from utils.logger import log
from .gmail_utils import find_nth

def parse_job_detail(body):
    """Parse html body and get job posting details
    Args:
      body: email html body
    Returns:
      String values which represent details of job post in JSON format.
    """
    try:
        link = body[find_nth(body, 'https://www.linkedin.com/comm/jobs/view/', 1): find_nth(body, '?trk', 1)]
        url = requests.get(link)
        html_text = url.text
        s = find_nth(html_text, '', 1)
        e = html_text.rfind('-->') + 10
        plain_data = html_text[s: e]
        plain_data = plain_data.replace('', '')
        soup = bs(plain_data, "html.parser")
        try:
            poster_information = soup.find('code', id='posterInformationModule')
            poster_information_json = poster_information.getText()
        except:
            poster_information_json = '{}'
        try:
            decorated_job_posting = soup.find('code', id='decoratedJobPostingModule')
            decorated_job_posting_json = decorated_job_posting.getText()
        except:
            decorated_job_posting_json = '{}'
        try:
            top_card_v2 = soup.find('code', id='topCardV2Module')
            top_card_v2_json = top_card_v2.getText()
        except:
            top_card_v2_json = '{}'

        return poster_information_json, decorated_job_posting_json, top_card_v2_json
    except Exception as e:
        log(traceback.format_exception(None, e, e.__traceback__), 'e')
        return '{}', '{}', '{}'

def get_access_token_with_code(code):
    try:
        if settings.DEBUG:
            redirect_uri = 'http://localhost:8080/action-linkedin-oauth2'
        else:
            redirect_uri = 'https://' + settings.SITE_URL + '/action-linkedin-oauth2'
        post_data = {'grant_type': 'authorization_code', 'code': code,
                     'redirect_uri': redirect_uri,
                     'client_id': os.environ.get('JOBHAX_LINKEDIN_CLIENT_KEY', ''),
                     'client_secret': os.environ.get('JOBHAX_LINKEDIN_CLIENT_SECRET', '')}
        log(post_data, 'e')
        response = requests.post('https://www.linkedin.com/uas/oauth2/accessToken',
                                 data=post_data, headers={'content-type': 'application/x-www-form-urlencoded'})
        json_res = json.loads(response.text)
        log(json_res, 'e')
        return json_res['access_token']
    except Exception as e:
        log(traceback.format_exception(None, e, e.__traceback__), 'e')
        return None


import datetime
import logging
from typing import Optional
from collections import defaultdict
from pathlib import Path
from betfairlightweight.resources.bettingresources import MarketBook, MarketCatalogue

from .. import config
from .blotter import Blotter
from ..execution.transaction import Transaction

logger = logging.getLogger(__name__)

class Market:
    """
    Market data structure to hold latest marketBook,
    marketCatalogue and various properties. Also
    allows order placement through the Transaction
    class.
    """

    def __init__(
        self,
        flumine,
        market_id: str,
        market_book: MarketBook,
        market_catalogue: MarketCatalogue = None,
    ):
        self.flumine = flumine
        self.market_id = market_id
        self.closed = False
        self.date_time_created = datetime.datetime.utcnow()
        self.date_time_closed = None
        self.market_book = market_book
        self.market_catalogue = market_catalogue
        self.update_market_catalogue = True
        self.orders_cleared = []
        self.market_cleared = []
        self.context = {"simulated": {}}  # data store (raceCard / scores etc)
        self.blotter = Blotter(market_id)
        self._transaction_id = 0

    def __call__(self, market_book: MarketBook):
        if self.market_book and market_book.version != self.market_book.version:
            self.update_market_catalogue = True
        self.market_book = market_book

    def open_market(self) -> None:
        self.closed = False
        self.orders_cleared = []
        self.market_cleared = []
        logger.info(
            "Market %s opened",
            self.market_id,
            extra=self.info,
        )

    def close_market(self) -> None:
        self.closed = True
        self.date_time_closed = datetime.datetime.utcnow()
        logger.info(
            "Market %s closed",
            self.market_id,
            extra=self.info,
        )

    def transaction(self, async_place_orders: bool = None, client=None) -> Transaction:
        if async_place_orders is None:
            async_place_orders = config.async_place_orders
        if client is None:
            client = self.flumine.clients.get_default()
        self._transaction_id += 1
        return Transaction(
            self,
            id_=self._transaction_id,
            async_place_orders=async_place_orders,
            client=client,
        )

    # order
    def place_order(
        self,
        order,
        market_version: int = None,
        execute: bool = True,
        force: bool = False,
        client=None,
    ) -> bool:
        with self.transaction(client=client) as t:
            return t.place_order(order, market_version, execute, force)

    def cancel_order(
        self, order, size_reduction: float = None, force: bool = False
    ) -> bool:
        with self.transaction(client=order.client) as t:
            return t.cancel_order(order, size_reduction, force)

    def update_order(
        self, order, new_persistence_type: str, force: bool = False
    ) -> bool:
        with self.transaction(client=order.client) as t:
            return t.update_order(order, new_persistence_type, force)

    def replace_order(
        self, order, new_price: float, market_version: int = None, force: bool = False
    ) -> bool:
        with self.transaction(client=order.client) as t:
            return t.replace_order(order, new_price, market_version, force)

    @property
    def event(self) -> dict:
        event = defaultdict(list)
        market_start_datetime = self.market_start_datetime
        event_type_id = self.event_type_id
        for market in self.flumine.markets.events[self.event_id]:
            if (
                market_start_datetime == market.market_start_datetime
                or event_type_id == "1"
            ):
                event[market.market_type].append(market)
        return event

    @property
    def event_type_id(self) -> str:
        if self.market_catalogue:
            return self.market_catalogue.event_type.id
        elif self.market_book:
            return self.market_book.market_definition.event_type_id

    @property
    def event_id(self) -> str:
        if self.market_catalogue:
            return self.market_catalogue.event.id
        elif self.market_book:
            return self.market_book.market_definition.event_id

    @property
    def market_type(self) -> str:
        if self.market_catalogue:
            return self.market_catalogue.description.market_type
        elif self.market_book:
            return self.market_book.market_definition.market_type

    @property
    def seconds_to_start(self) -> float:
        return (self.market_start_datetime - datetime.datetime.utcnow()).total_seconds()

    @property
    def elapsed_seconds_closed(self) -> Optional[float]:
        if self.closed and self.date_time_closed:
            return (datetime.datetime.utcnow() - self.date_time_closed).total_seconds()

    @property
    def market_start_datetime(self):
        if self.market_book:
            return self.market_book.market_definition.market_time
        elif self.market_catalogue:
            return self.market_catalogue.market_start_time
        else:
            return datetime.datetime.utcfromtimestamp(0)

    @property
    def market_start_hour_minute(self) -> Optional[str]:
        msd = self.market_start_datetime
        if msd:
            return "{0:02d}{1:02d}".format(msd.hour, msd.minute)

    @property
    def event_name(self) -> Optional[str]:
        if self.market_catalogue:
            return self.market_catalogue.event.name
        elif self.market_book:
            return self.market_book.market_definition.event_name

    @property
    def country_code(self) -> Optional[str]:
        if self.market_catalogue:
            return self.market_catalogue.event.country_code
        elif self.market_book:
            return self.market_book.market_definition.country_code

    @property
    def venue(self) -> Optional[str]:
        if self.market_catalogue:
            return self.market_catalogue.event.venue
        elif self.market_book:
            return self.market_book.market_definition.venue

    @property
    def race_type(self) -> Optional[str]:
        if self.market_catalogue:
            return self.market_catalogue.description.race_type
        elif self.market_book:
            return self.market_book.market_definition.race_type

    @property
    def status(self) -> Optional[str]:
        if self.market_book:
            return self.market_book.status

    def cleared(self, client) -> dict:
        orders = self.blotter.client_orders(client, matched_only=True)
        profit = round(sum([order.profit for order in orders]), 2)
        return {
            "marketId": self.market_id,
            "eventId": self.event_id,
            "eventTypeId": self.event_type_id,
            "customerStrategyRef": config.customer_strategy_ref,
            "lastMatchedDate": None,
            "placedDate": None,
            "settledDate": None,
            "betCount": len(orders),
            "betOutcome": "WON" if profit >= 0 else "LOST",
            "commission": round(max(profit * client.commission_base, 0), 2),
            "profit": profit,
        }

    @property
    def info(self) -> dict:
        return {
            "market_id": self.market_id,
            "event_id": self.event_id,
            "event_type_id": self.event_type_id,
            "event_name": self.event_name,
            "market_type": self.market_type,
            "market_start_datetime": str(self.market_start_datetime),
            "country_code": self.country_code,
            "venue": self.venue,
            "race_type": self.race_type,
            "orders_cleared": self.orders_cleared,
            "market_cleared": self.market_cleared,
            "closed": self.closed,
        }